def ApplyPowerSchedule(self,kCustomer):
from datetime import datetime, date
import sys, time, string
import boto.ec2
import boto.vpc
fullInstancesSchedList = []
taskInstancesSchedListStartup = []
taskInstancesSchedListStop = []
self.showInitialMsg()
ObjCustomer = clsCustomer(kCustomer)
self.printMsg ("","Connecting ...")
self.__awsConnection = boto.ec2.connect_to_region(
ObjCustomer.Region,
aws_access_key_id=ObjCustomer.Access_key,
aws_secret_access_key=ObjCustomer.Secret_key)
self.printMsg ("","---> " + "Connected")
self.printMsg ("","Getting schedule details...")
#reservations = self.__awsConnection.get_all_reservations()
instances = [i for r in self.__awsConnection.get_all_reservations() for i in r.instances]
for instance in instances:
ObjInstanceSched = clsInstanceSched()
# Check if the instance has both of the required tang + the flag to enable scheduling
schedValues=""
if instance.tags.get('bombo_autosched:SCHEDULE') :
schedValues = instance.tags.get('bombo_autosched:SCHEDULE').upper().split(",")
ObjInstanceSched.schedEnabled = (True if schedValues[0] == "Y" else False)
ObjInstanceSched.start = schedValues[1].split("-")[0].replace(":"," ")
ObjInstanceSched.end = schedValues[1].split("-")[1].replace(":"," ")
ObjInstanceSched.day = int(schedValues[2])
ObjInstanceSched.deps = str(instance.tags.get('bombo_autosched:DEPS'))
#Adding instance even if SCHEDULING is not enabled
ObjInstanceSched.instance = instance
#Populating full Instances list with Scheduling option info
fullInstancesSchedList.append(ObjInstanceSched)
if ObjInstanceSched.schedEnabled:
# The Schedule wants to be applied. Extract the timerange and power on/off
try:
# Check if the current time + day of week is inside the bombo_autosched:SCHEDULE range
if date.isoweekday(date.today()) <= ObjInstanceSched.day and ObjInstanceSched.start <= time.strftime("%H %M",time.localtime()) < ObjInstanceSched.end :
# It is inside the working schedule range, power it up if it is shutdown
if instance.state == "stopped" :
#Adding reservation to list
taskInstancesSchedListStartup.append(ObjInstanceSched)
else:
# It is currently outside of the scheduled working hours, shut it down if it is running
if instance.state == "running" :
taskInstancesSchedListStop.append(ObjInstanceSched)
# Capture an error if bombo_autosched:SCHEDULE isn't formatted correctly
except IndexError:
self.printMsg ("","###> Error with the tag bombo_autosched:SCHEDULE (expecting <Y/N>,hh:mm-hh:mm,<days>), found: " + instance.tags.get('bombo_autosched:SCHEDULE') + " on instance: " + instance.id )
# # tag all the instances with the Auto Scheduling tags if it does not already exist, but set it to be ignored.
# if not instance.tags.get('bombo_autosched:SCHEDULE'):
# self.printMsg ("","Adding tag SCHEDULE to instance " + instance.id)
# self.__awsConnection.create_tags([instance.id], {"bombo_autosched:SCHEDULE": "N,08:00-20:00,5,N,08:00-20:00,5,A"})
# if not instance.tags.get('bombo_autosched:DEPS'):
# self.printMsg ("","Adding tag DEPS to instance " + instance.id)
# self.__awsConnection.create_tags([instance.id], {"bombo_autosched:DEPS": ""})
ObjScheduling = clsScheduling()
listToStart=ObjScheduling.getScheduledListStartup(taskInstancesSchedListStartup,fullInstancesSchedList)
self.printMsg ("","Found " + str(len(listToStart)) + " instances to start!")
for i in listToStart:
#Check if the dependency can be started automatically
if i.isADeps and not i.schedEnabled:
self.printMsg ("","###> A depencendy instance was not scheduled for startup automatically: " + i.instance.id,True,True)
self.printMsg ("","Starting instance [" + i.instance.id + "]")
tmpInstance=self.__awsConnection.start_instances(i.instance.id)
if i.isADeps:
self.printMsg ("","This instance is a depenency, so I'm waiting for the instance to be up and running")
ObjScheduling.checkSchedInstanceState(tmpInstance[0])
listToStop=ObjScheduling.getScheduledListStop(taskInstancesSchedListStop,fullInstancesSchedList)
self.printMsg ("","Found " + str(len(listToStop)) + " instances to stop!")
for i in listToStop:
self.printMsg ("","Stopping instance [" + i.instance.id + "]")
tmpInstance=self.__awsConnection.stop_instances(i.instance.id)
self.printMsg ("","Waiting for the instance to be stopped")
ObjScheduling.checkSchedInstanceState(tmpInstance[0])
self.printMsg ("","--->Finished with the scheduling")
def __getObjCustomer(self,kCustomerId):
self.CustomerId = kCustomerId
self.Customer = clsCustomer(kCustomerId)
def BackupInstance(self,kCustomerId,kRegion,kInstanceId,kKeepInstanceOn = False, kFlushOldBackup = False, kKeepHistoricals = False):
from datetime import datetime, timedelta
import sys, time
import boto.ec2
import boto.vpc
# Will tag all the volumes on the instances requested, then make a snapshot of the volumes, then tag the snapshots, then purge
self.showInitialMsg()
if kKeepInstanceOn:
self.printMsg ("","###> You choose to backup the instance without switching it off, please check when done if it was completed successfully!")
ObjCustomer = clsCustomer(kCustomerId)
self.printMsg ("","Connecting ...")
try:
self.__awsConnection = boto.ec2.connect_to_region(
kRegion,
aws_access_key_id=ObjCustomer.Access_key,
aws_secret_access_key=ObjCustomer.Secret_key)
except:
self.printMsg("AWS","Ops! I found problems during connection..." + str(sys.exc_info()[0]),True,True)
self.printMsg ("","---> " + "Connected")
self.printMsg ("","Getting instances details...")
if kInstanceId == "all":
instances = [i for r in self.__awsConnection.get_all_reservations() for i in r.instances]
else:
instances = [i for r in self.__awsConnection.get_all_instances(instance_ids=[kInstanceId]) for i in r.instances]
self.printMsg ("","---> " + str(len(instances)) + " Found")
VolumesSnapshotMatchList = []
FailedVolumesSnapshot = []
FailedDeleteSnapshot = []
pointer=0
for instance in instances:
pointer+=1
self.printMsg ("","###--------------------------------------------------------------------------------###")
self.printMsg ("","Working on " + instance.id + " [" + str(pointer) + " of " + str(len(instances)) + "]")
self.printMsg ("","Getting volumes...")
vols = self.__awsConnection.get_all_volumes(filters={'attachment.instance-id': instance.id})
self.printMsg ("","---> " + str(len(vols)) + " Volumes found")
self.printMsg ("","Tagging instance...")
self.__awsConnection.create_tags([instance.id], {"bombo_backup": datetime.today().strftime('%d-%m-%Y %H:%M:%S')})
self.printMsg ("","---> " + "bombo_backup - " + datetime.today().strftime('%d-%m-%Y %H:%M:%S'))
if kKeepInstanceOn == False:
if instance.update() != "stopped":
self.printMsg ("","Stopping the instance...")
instances_to_stop = self.__awsConnection.stop_instances(instance_ids=instance.id)
counter = 0
spinner = self.spinning_cursor()
while counter != len(instances_to_stop):
sys.stdout.write(spinner.next())
sys.stdout.flush()
sys.stdout.write('\b')
for instance in instances_to_stop:
if instance.update() == "stopped":
counter += 1
time.sleep(2)
else:
self.printMsg ("","Instance is already stopped...")
self.printMsg ("","---> Done")
for vol in vols:
self.printMsg ("","Tagging volume " + vol.id + "...")
vol.add_tag("bombo_backup:INSTANCE", instance.id)
vol.add_tag("bombo_backup:STATUS", vol.attach_data.status)
vol.add_tag("bombo_backup:DEVICE", vol.attach_data.device)
vol.add_tag("bombo_backup:DATE", datetime.today().strftime('%d-%m-%Y %H:%M:%S'))
self.printMsg ("","Snapshot volume " + vol.id + "...")
try:
snapshot = self.__awsConnection.create_snapshot(vol.id, "INSTANCE: " + instance.id + " - DEVICE:" + vol.attach_data.device)
VolumesSnapshotMatchList.append([vol,snapshot])
except:
self.printMsg ("", "ERROR trying to create snapshot for instance " + str(instance.id) + " - DEVICE:" + vol.attach_data.device)
FailedVolumesSnapshot.append([vol])
self.printMsg ("","---> Done with [" + str(vol.id) + "] => " + "INSTANCE: " + instance.id + " - DEVICE:" + vol.attach_data.device)
self.printMsg ("","###--------------------------------------------------------------------------------###")
self.printMsg ("","Waiting for the snapshots to be ready...")
counter = 0
progress = 0
singleValue = ""
spinner = self.spinning_cursor()
while progress < (100 * len(VolumesSnapshotMatchList)):
progress = 0
for VolumesSnapshotMatch in VolumesSnapshotMatchList:
singleValue = VolumesSnapshotMatch[1].update()
if len(singleValue)==0:
singleValue="0%"
progress = progress + int(singleValue[:-1])
time.sleep(2)
sys.stdout.flush()
sys.stdout.write('\r')
sys.stdout.write(spinner.next() + " Waiting for " + str(len(VolumesSnapshotMatchList)) + " snapshots => Progress: " + str(progress / len(VolumesSnapshotMatchList)) + "%")
sys.stdout.write('\n')
self.printMsg ("","---> Snapshots ready...")
self.printMsg ("","Tagging Snapshots...")
for VolumesSnapshotMatch in VolumesSnapshotMatchList:
VolumesSnapshotMatch[1].add_tag("bombo_backup:DATE", datetime.today().strftime('%d-%m-%Y %H:%M:%S'))
VolumesSnapshotMatch[1].add_tag("bombo_backup:INSTANCE", VolumesSnapshotMatch[0].tags.get('bombo_backup:INSTANCE'))
VolumesSnapshotMatch[1].add_tag("bombo_backup:DEVICE", VolumesSnapshotMatch[0].attach_data.device)
VolumesSnapshotMatch[1].add_tag("Name", "Backup of: " + VolumesSnapshotMatch[0].tags.get('bombo_backup:INSTANCE'))
self.printMsg ("","Snapshot " + str(VolumesSnapshotMatch[1].id))
self.printMsg ("","---> " + "bombo_backup:DATE " + str(datetime.today().strftime('%d-%m-%Y %H:%M:%S')))
self.printMsg ("","---> " + "bombo_backup:INSTANCE " + str(instance.id))
self.printMsg ("","---> " + "bombo_backup:DEVICE " + VolumesSnapshotMatch[0].attach_data.device)
#
# Snapshots have been made, now to purge the old ones
if kFlushOldBackup:
deletion_counter = 0
size_counter = 0
delete_time = datetime.utcnow() - timedelta(days=ObjCustomer.BckVolumesRetention)
historical_age = datetime.utcnow() - timedelta(days=ObjCustomer.HistoricalRetention)
if kKeepHistoricals:
self.printMsg ("","Deleting any snapshots older than {days} days (except for those created on the 1st of the month which are kept for {history} days)".format(days=ObjCustomer.BckVolumesRetention,history=ObjCustomer.HistoricalRetention))
else:
self.printMsg ("","Deleting any snapshots older than {days} days".format(days=ObjCustomer.BckVolumesRetention))
snapshots = self.__awsConnection.get_all_snapshots()
for snapshot in snapshots:
if 'bombo_backup:INSTANCE' in snapshot.tags:
if ((snapshot.tags.get('bombo_backup:INSTANCE') == kInstanceId) or (str(kInstanceId).lower() == "all")):
#Only purge specified instance snapshot OR all of the bombo snapshots
start_time = datetime.strptime(snapshot.start_time,'%Y-%m-%dT%H:%M:%S.000Z')
if kKeepHistoricals:
# We want to keep a monthly backup from the 1st of the month until the maximum age as per configured by the customer
if (start_time < delete_time and start_time.day <> 1):
# If the backup is older than retention period, but it wasn't created on the 1st of the month, delete it
print ('Deleting {id}'.format(id=snapshot.id)) + " made on " + str(snapshot.tags.get('bombo_backup:DATE')) + " attached to " + str(snapshot.tags.get('bombo_backup:INSTANCE')) + " mounted on " + str(snapshot.tags.get('bombo_backup:DEVICE'))
deletion_counter = deletion_counter + 1
size_counter = size_counter + snapshot.volume_size
try:
snapshot.delete(dry_run=False)
except:
FailedDeleteSnapshot.append(snapshot.id)
elif start_time.day == 1 and start_time < historical_age:
# Else if the snapshot was on the 1st of the month AND it is older than the historical retention period it can be deleted.
print ('Deleting monthly {id}'.format(id=snapshot.id)) + " made on " + str(snapshot.tags.get('bombo_backup:DATE')) + " attached to " + str(snapshot.tags.get('bombo_backup:INSTANCE')) + " mounted on " + str(snapshot.tags.get('bombo_backup:DEVICE'))
deletion_counter = deletion_counter + 1
size_counter = size_counter + snapshot.volume_size
try:
snapshot.delete(dry_run=False)
except:
FailedDeleteSnapshot.append(snapshot.id)
else:
# If the backup is older than retention period, delete it as we are not interested in keeping monthlies
if start_time < delete_time:
print ('Deleting {id}'.format(id=snapshot.id)) + " made on " + str(snapshot.tags.get('bombo_backup:DATE')) + " attached to " + str(snapshot.tags.get('bombo_backup:INSTANCE')) + " mounted on " + str(snapshot.tags.get('bombo_backup:DEVICE'))
deletion_counter = deletion_counter + 1
size_counter = size_counter + snapshot.volume_size
try:
snapshot.delete(dry_run=False)
except:
FailedDeleteSnapshot.append(snapshot.id)
print 'Deleted {number} snapshots totalling {size} GB'.format(number=deletion_counter,size=size_counter)
self.printMsg ("","Hopefully everything went well......")
#
# Print any snapshots that failed
if FailedVolumesSnapshot:
self.printMsg ("", "Unfortunately, things did not go so well......")
for FailedVolume in FailedVolumesSnapshot:
self.printMsg ("", "ERROR trying to create snapshot for instance " + FailedVolume[0].tags.get('bombo_backup:INSTANCE') + " , Device " + FailedVolume[0].tags.get('bombo_backup:DEVICE'))
print ("")
for FailedDelete in FailedDeleteSnapshot:
self.printMsg ("", "ERROR trying to delete snapshot " + FailedDelete)
def CopyInstance(self,kCustomerId,kRegion,kInstanceId,kSubnet,kKeepInstanceOn = False):
from datetime import datetime
import sys, time
import boto.ec2
import boto.vpc
self.showInitialMsg()
if kKeepInstanceOn:
self.printMsg ("","###> You choose to copy the instance without switching it off, please check when done if it was completed successfully!")
ObjCustomer = clsCustomer(kCustomerId)
self.printMsg ("","Connecting ...")
self.__awsConnection = boto.ec2.connect_to_region(
kRegion,
aws_access_key_id=ObjCustomer.Access_key,
aws_secret_access_key=ObjCustomer.Secret_key)
self.printMsg ("","---> " + "Connected")
self.printMsg ("","Getting instance details...")
reservations = self.__awsConnection.get_all_instances(instance_ids=[kInstanceId])
instance = reservations[0].instances[0]
self.printMsg ("","---> " + instance.id)
self.printMsg ("","Getting subnet details...")
self.__awsVpcConnection = boto.vpc.connect_to_region(
kRegion,
aws_access_key_id=ObjCustomer.Access_key,
aws_secret_access_key=ObjCustomer.Secret_key)
subnets = self.__awsVpcConnection.get_all_subnets(subnet_ids=[kSubnet])
old_vpc_id = subnets[0].vpc_id
old_azone = subnets[0].availability_zone
self.printMsg ("","---> Found " + kSubnet + " linked to " + old_vpc_id + " in " + old_azone)
if kKeepInstanceOn == False:
self.printMsg ("","Stopping the instance...")
instances_to_stop = self.__awsConnection.stop_instances(instance_ids=kInstanceId)
counter = 0
spinner = self.spinning_cursor()
while counter != len(instances_to_stop):
sys.stdout.write(spinner.next())
sys.stdout.flush()
sys.stdout.write('\b')
for instance in instances_to_stop:
if instance.update() == "stopped":
counter += 1
time.sleep(2)
self.printMsg ("","---> Done")
self.printMsg ("","Getting volumes...")
vols = self.__awsConnection.get_all_volumes(filters={'attachment.instance-id': instance.id})
self.printMsg ("","---> " + str(len(vols)) + " Volumes found")
self.printMsg ("","Tagging instance...")
self.__awsConnection.create_tags([instance.id], {"bombo_moving": kSubnet + " - " + datetime.today().strftime('%d-%m-%Y %H:%M:%S')})
self.printMsg ("","---> " + kSubnet + " - " + datetime.today().strftime('%d-%m-%Y %H:%M:%S'))
#self.printMsg ("","Copying tags...")
#TagsList = []
#TagsList = setTagsToInstance([instance.id])
#self.printMsg ("","Finished copying tags")
VolumesSnapshotMatchList = []
for vol in vols:
self.printMsg ("","Tagging volume " + vol.id + "...")
if instance.tags.get('Name') :
vol.add_tag("Name", "### Copy of :" + instance.tags.get('Name') + " ###")
else:
vol.add_tag("Name", "### Copied ###")
vol.add_tag("bombo_moving:INSTANCE", instance.id)
vol.add_tag("bombo_moving:STATUS", vol.attach_data.status)
vol.add_tag("bombo_moving:DEVICE", vol.attach_data.device)
self.printMsg ("","Snapshot volume " + vol.id + "...")
snapshot = self.__awsConnection.create_snapshot(vol.id, "INSTANCE: " + instance.id + " - DEVICE:" + vol.attach_data.device)
VolumesSnapshotMatchList.append([vol,snapshot])
self.printMsg ("","---> Done with [" + str(vol.id) + "] => " + "INSTANCE: " + instance.id + " - DEVICE:" + vol.attach_data.device)
self.printMsg ("","Waiting for the snapshots to be ready...")
counter = 0
progress = 0
singleValue = ""
spinner = self.spinning_cursor()
while progress < (100 * len(VolumesSnapshotMatchList)):
progress = 0
for VolumesSnapshotMatch in VolumesSnapshotMatchList:
singleValue = VolumesSnapshotMatch[1].update()
if len(singleValue)==0:
singleValue="0%"
progress = progress + int(singleValue[:-1])
time.sleep(2)
sys.stdout.flush()
sys.stdout.write('\r')
sys.stdout.write(spinner.next() + " Waiting for " + str(len(VolumesSnapshotMatchList)) + " snapshots => Progress: " + str(progress / len(VolumesSnapshotMatchList)) + "%")
sys.stdout.write('\n')
self.printMsg ("","---> Snapshots ready...")
NewInstanceVolumeList = []
self.printMsg ("","Creating volumes from snapshot...")
for VolumesSnapshotMatch in VolumesSnapshotMatchList:
new_vol_tmp = self.__awsConnection.create_volume(
size=VolumesSnapshotMatch[0].size,
volume_type = VolumesSnapshotMatch[0].type,
snapshot=VolumesSnapshotMatch[1],
zone=old_azone
)
new_vol_tmp.add_tag("bombo_moving:DATE", datetime.today().strftime('%d-%m-%Y %H:%M:%S'))
new_vol_tmp.add_tag("bombo_moving:SOURCE", kInstanceId + " -> " + VolumesSnapshotMatch[0].attach_data.device)
NewInstanceVolumeList.append([VolumesSnapshotMatch[0].attach_data.device,new_vol_tmp])
self.printMsg ("","---> [" + VolumesSnapshotMatch[0].attach_data.device + "] - " + VolumesSnapshotMatch[0].type + ", " + str(VolumesSnapshotMatch[0].size) + " GB created")
self.printMsg ("","---> Volumes ready...")
self.printMsg ("","Check AMI availability " + instance.image_id + "...")
ObjInstance = clsInstance(instance.id,ObjCustomer,instance)
if ObjInstance.checkAmiAvailility(self.__awsConnection):
self.printMsg ("","---> AMI Available")
ObjInstance.Ami = instance.image_id
else:
self.printMsg ("","###> AMI NOT Available, searching for a similar AMI [" + instance.architecture +" - " + instance.virtualization_type +" - " + instance.hypervisor +" - " + instance.root_device_type + "]")
ObjInstance.Ami = self.searchSimilarAMI(self.__awsConnection,instance)
self.printMsg ("","Launching new instance on " + kSubnet + "...")
reservation = self.__awsConnection.run_instances(
image_id = ObjInstance.Ami,
instance_type = instance.instance_type,
key_name = instance.key_name,
#private_ip_address = "10.0.34.100",
#security_group_ids = kSingleLaunch.SecGroups,
subnet_id=kSubnet,
dry_run=False,
)
counter = 0
spinner = self.spinning_cursor()
while counter != len(reservation.instances):
sys.stdout.write(spinner.next())
sys.stdout.flush()
sys.stdout.write('\b')
for instance in reservation.instances:
if instance.update() != "pending":
counter += 1
time.sleep(2) #Must wait even when the instance is ready, otherwise we got instance not found by AWS
self.printMsg ("","---> [" + reservation.instances[0].id + "] => Instance launched")
self.printMsg ("","Tagging new instance...")
self.__awsConnection.create_tags([instance.id], {"bombo_moving:SOURCE_INSTANCE":kInstanceId})
self.__awsConnection.create_tags([instance.id], {"bombo_moving:DATE":datetime.today().strftime('%d-%m-%Y %H:%M:%S')})
# self.printMsg ("","Extracting tags from source instance...")
# TagsList = self.getTagsFromInstance(kInstanceId)
# self.printMsg ("","Applying tags to the new instance...")
# self.setTagsToInstance(TagsList, instance.id)
self.printMsg ("","---> New instance tagged...")
self.printMsg ("","Stopping NEW instance...")
self.__awsConnection.stop_instances(
instance_ids=reservation.instances[0].id,
force=True)
counter = 0
spinner = self.spinning_cursor()
while counter != len(reservation.instances):
sys.stdout.write(spinner.next())
sys.stdout.flush()
sys.stdout.write('\b')
for instance in reservation.instances:
if instance.update() == "stopped":
counter += 1
time.sleep(2)
self.printMsg ("","---> Done")
self.printMsg ("","Getting volumes list of the new instance...")
new_instance_vols = self.__awsConnection.get_all_volumes(filters={'attachment.instance-id': reservation.instances[0].id})
self.printMsg ("","---> " + str(len(vols)) + " volumes found, deleting all..")
for new_instance_vol in new_instance_vols:
new_instance_vol.detach(
force=True)
self.printMsg ("","---> " + str(new_instance_vol.id) + " detached....")
new_instance_vol.delete
self.printMsg ("","---> " + str(new_instance_vol.id) + " and now deleted!")
counter = 0
spinner = self.spinning_cursor()
while counter < len(new_instance_vols):
sys.stdout.write(spinner.next())
sys.stdout.flush()
sys.stdout.write('\b')
for vol_check_detach in new_instance_vols:
if vol_check_detach.update() == "available":
counter += 1
time.sleep(2)
self.printMsg ("","Attaching volumes to the new instance...")
for NewInstanceVolume in NewInstanceVolumeList:
self.__awsConnection.attach_volume(
instance_id = reservation.instances[0].id,
volume_id = NewInstanceVolume[1].id,
device = NewInstanceVolume[0])
self.printMsg ("","---> " + NewInstanceVolume[0] + " attached to " + str(reservation.instances[0].id))
self.printMsg ("","Starting the NEW instance...")
self.__awsConnection.start_instances(instance_ids=reservation.instances[0].id)
self.printMsg ("","---> Done")
self.printMsg ("","Hopefully everything went well......")
def ApplyPowerSchedule(self, kCustomer):
from datetime import datetime, date
import sys, time, string
import boto.ec2
import boto.vpc
fullInstancesSchedList = []
taskInstancesSchedListStartup = []
taskInstancesSchedListStop = []
self.showInitialMsg()
ObjCustomer = clsCustomer(kCustomer)
self.printMsg("", "Connecting ...")
self.__awsConnection = boto.ec2.connect_to_region(
ObjCustomer.Region,
aws_access_key_id=ObjCustomer.Access_key,
aws_secret_access_key=ObjCustomer.Secret_key)
self.printMsg("", "---> " + "Connected")
self.printMsg("", "Getting schedule details...")
#reservations = self.__awsConnection.get_all_reservations()
instances = [
i for r in self.__awsConnection.get_all_reservations()
for i in r.instances
]
for instance in instances:
ObjInstanceSched = clsInstanceSched()
# Check if the instance has both of the required tang + the flag to enable scheduling
schedValues = ""
if instance.tags.get('bombo_autosched:SCHEDULE'):
schedValues = instance.tags.get(
'bombo_autosched:SCHEDULE').upper().split(",")
ObjInstanceSched.schedEnabled = (True if schedValues[0] == "Y"
else False)
ObjInstanceSched.start = schedValues[1].split("-")[0].replace(
":", " ")
ObjInstanceSched.end = schedValues[1].split("-")[1].replace(
":", " ")
ObjInstanceSched.day = int(schedValues[2])
ObjInstanceSched.deps = str(
instance.tags.get('bombo_autosched:DEPS'))
#Adding instance even if SCHEDULING is not enabled
ObjInstanceSched.instance = instance
#Populating full Instances list with Scheduling option info
fullInstancesSchedList.append(ObjInstanceSched)
if ObjInstanceSched.schedEnabled:
# The Schedule wants to be applied. Extract the timerange and power on/off
try:
# Check if the current time + day of week is inside the bombo_autosched:SCHEDULE range
if date.isoweekday(
date.today()
) <= ObjInstanceSched.day and ObjInstanceSched.start <= time.strftime(
"%H %M", time.localtime()) < ObjInstanceSched.end:
# It is inside the working schedule range, power it up if it is shutdown
if instance.state == "stopped":
#Adding reservation to list
taskInstancesSchedListStartup.append(
ObjInstanceSched)
else:
# It is currently outside of the scheduled working hours, shut it down if it is running
if instance.state == "running":
taskInstancesSchedListStop.append(ObjInstanceSched)
# Capture an error if bombo_autosched:SCHEDULE isn't formatted correctly
except IndexError:
self.printMsg(
"",
"###> Error with the tag bombo_autosched:SCHEDULE (expecting <Y/N>,hh:mm-hh:mm,<days>), found: "
+ instance.tags.get('bombo_autosched:SCHEDULE') +
" on instance: " + instance.id)
# # tag all the instances with the Auto Scheduling tags if it does not already exist, but set it to be ignored.
# if not instance.tags.get('bombo_autosched:SCHEDULE'):
# self.printMsg ("","Adding tag SCHEDULE to instance " + instance.id)
# self.__awsConnection.create_tags([instance.id], {"bombo_autosched:SCHEDULE": "N,08:00-20:00,5,N,08:00-20:00,5,A"})
# if not instance.tags.get('bombo_autosched:DEPS'):
# self.printMsg ("","Adding tag DEPS to instance " + instance.id)
# self.__awsConnection.create_tags([instance.id], {"bombo_autosched:DEPS": ""})
ObjScheduling = clsScheduling()
listToStart = ObjScheduling.getScheduledListStartup(
taskInstancesSchedListStartup, fullInstancesSchedList)
self.printMsg(
"", "Found " + str(len(listToStart)) + " instances to start!")
for i in listToStart:
#Check if the dependency can be started automatically
if i.isADeps and not i.schedEnabled:
self.printMsg(
"",
"###> A depencendy instance was not scheduled for startup automatically: "
+ i.instance.id, True, True)
self.printMsg("", "Starting instance [" + i.instance.id + "]")
tmpInstance = self.__awsConnection.start_instances(i.instance.id)
if i.isADeps:
self.printMsg(
"",
"This instance is a depenency, so I'm waiting for the instance to be up and running"
)
ObjScheduling.checkSchedInstanceState(tmpInstance[0])
listToStop = ObjScheduling.getScheduledListStop(
taskInstancesSchedListStop, fullInstancesSchedList)
self.printMsg("",
"Found " + str(len(listToStop)) + " instances to stop!")
for i in listToStop:
self.printMsg("", "Stopping instance [" + i.instance.id + "]")
tmpInstance = self.__awsConnection.stop_instances(i.instance.id)
self.printMsg("", "Waiting for the instance to be stopped")
ObjScheduling.checkSchedInstanceState(tmpInstance[0])
self.printMsg("", "--->Finished with the scheduling")
def BackupInstance(self,
kCustomerId,
kRegion,
kInstanceId,
kKeepInstanceOn=False,
kFlushOldBackup=False,
kKeepHistoricals=False):
from datetime import datetime, timedelta
import sys, time
import boto.ec2
import boto.vpc
# Will tag all the volumes on the instances requested, then make a snapshot of the volumes, then tag the snapshots, then purge
self.showInitialMsg()
if kKeepInstanceOn:
self.printMsg(
"",
"###> You choose to backup the instance without switching it off, please check when done if it was completed successfully!"
)
ObjCustomer = clsCustomer(kCustomerId)
self.printMsg("", "Connecting ...")
try:
self.__awsConnection = boto.ec2.connect_to_region(
kRegion,
aws_access_key_id=ObjCustomer.Access_key,
aws_secret_access_key=ObjCustomer.Secret_key)
except:
self.printMsg(
"AWS", "Ops! I found problems during connection..." +
str(sys.exc_info()[0]), True, True)
self.printMsg("", "---> " + "Connected")
self.printMsg("", "Getting instances details...")
if kInstanceId == "all":
instances = [
i for r in self.__awsConnection.get_all_reservations()
for i in r.instances
]
else:
instances = [
i for r in self.__awsConnection.get_all_instances(
instance_ids=[kInstanceId]) for i in r.instances
]
self.printMsg("", "---> " + str(len(instances)) + " Found")
VolumesSnapshotMatchList = []
FailedVolumesSnapshot = []
FailedDeleteSnapshot = []
pointer = 0
for instance in instances:
pointer += 1
self.printMsg(
"",
"###--------------------------------------------------------------------------------###"
)
self.printMsg(
"", "Working on " + instance.id + " [" + str(pointer) +
" of " + str(len(instances)) + "]")
self.printMsg("", "Getting volumes...")
vols = self.__awsConnection.get_all_volumes(
filters={'attachment.instance-id': instance.id})
self.printMsg("", "---> " + str(len(vols)) + " Volumes found")
self.printMsg("", "Tagging instance...")
self.__awsConnection.create_tags([instance.id], {
"bombo_backup":
datetime.today().strftime('%d-%m-%Y %H:%M:%S')
})
self.printMsg(
"", "---> " + "bombo_backup - " +
datetime.today().strftime('%d-%m-%Y %H:%M:%S'))
if kKeepInstanceOn == False:
if instance.update() != "stopped":
self.printMsg("", "Stopping the instance...")
instances_to_stop = self.__awsConnection.stop_instances(
instance_ids=instance.id)
counter = 0
spinner = self.spinning_cursor()
while counter != len(instances_to_stop):
sys.stdout.write(spinner.next())
sys.stdout.flush()
sys.stdout.write('\b')
for instance in instances_to_stop:
if instance.update() == "stopped":
counter += 1
time.sleep(2)
else:
self.printMsg("", "Instance is already stopped...")
self.printMsg("", "---> Done")
for vol in vols:
if ObjCustomer.TagVolumesOnBackup == True:
self.printMsg("", "Tagging volume " + vol.id + "...")
vol.add_tag("bombo_backup:INSTANCE", instance.id)
vol.add_tag("bombo_backup:STATUS", vol.attach_data.status)
vol.add_tag("bombo_backup:DEVICE", vol.attach_data.device)
vol.add_tag("bombo_backup:DATE",
datetime.today().strftime('%d-%m-%Y %H:%M:%S'))
self.printMsg("", "Snapshot volume " + vol.id + "...")
try:
snapshot = self.__awsConnection.create_snapshot(
vol.id, "INSTANCE: " + instance.id + " - DEVICE:" +
vol.attach_data.device)
VolumesSnapshotMatchList.append([vol, snapshot])
except:
self.printMsg(
"", "ERROR trying to create snapshot for instance " +
str(instance.id) + " - DEVICE:" +
vol.attach_data.device)
FailedVolumesSnapshot.append([vol])
self.printMsg(
"",
"---> Done with [" + str(vol.id) + "] => " + "INSTANCE: " +
instance.id + " - DEVICE:" + vol.attach_data.device)
self.printMsg(
"",
"###--------------------------------------------------------------------------------###"
)
self.printMsg("", "Waiting for the snapshots to be ready...")
counter = 0
progress = 0
singleValue = ""
spinner = self.spinning_cursor()
while progress < (100 * len(VolumesSnapshotMatchList)):
progress = 0
for VolumesSnapshotMatch in VolumesSnapshotMatchList:
singleValue = VolumesSnapshotMatch[1].update()
if len(singleValue) == 0:
singleValue = "0%"
progress = progress + int(singleValue[:-1])
time.sleep(2)
sys.stdout.flush()
sys.stdout.write('\r')
sys.stdout.write(spinner.next() + " Waiting for " +
str(len(VolumesSnapshotMatchList)) +
" snapshots => Progress: " +
str(progress / len(VolumesSnapshotMatchList)) +
"%")
sys.stdout.write('\n')
self.printMsg("", "---> Snapshots ready...")
self.printMsg("", "Tagging Snapshots...")
for VolumesSnapshotMatch in VolumesSnapshotMatchList:
VolumesSnapshotMatch[1].add_tag(
"bombo_backup:DATE",
datetime.today().strftime('%d-%m-%Y %H:%M:%S'))
VolumesSnapshotMatch[1].add_tag(
"bombo_backup:INSTANCE",
VolumesSnapshotMatch[0].tags.get('bombo_backup:INSTANCE'))
VolumesSnapshotMatch[1].add_tag(
"bombo_backup:DEVICE",
VolumesSnapshotMatch[0].attach_data.device)
VolumesSnapshotMatch[1].add_tag(
"Name", "Backup of: " +
VolumesSnapshotMatch[0].tags.get('bombo_backup:INSTANCE'))
self.printMsg("", "Snapshot " + str(VolumesSnapshotMatch[1].id))
self.printMsg(
"", "---> " + "bombo_backup:DATE " +
str(datetime.today().strftime('%d-%m-%Y %H:%M:%S')))
self.printMsg(
"", "---> " + "bombo_backup:INSTANCE " + str(instance.id))
self.printMsg(
"", "---> " + "bombo_backup:DEVICE " +
VolumesSnapshotMatch[0].attach_data.device)
#
# Snapshots have been made, now to purge the old ones
if kFlushOldBackup:
deletion_counter = 0
size_counter = 0
delete_time = datetime.utcnow() - timedelta(
days=ObjCustomer.BckVolumesRetention)
historical_age = datetime.utcnow() - timedelta(
days=ObjCustomer.HistoricalRetention)
if kKeepHistoricals:
self.printMsg(
"",
"Deleting any snapshots older than {days} days (except for those created on the 1st of the month which are kept for {history} days)"
.format(days=ObjCustomer.BckVolumesRetention,
history=ObjCustomer.HistoricalRetention))
else:
self.printMsg(
"", "Deleting any snapshots older than {days} days".format(
days=ObjCustomer.BckVolumesRetention))
snapshots = self.__awsConnection.get_all_snapshots()
for snapshot in snapshots:
if 'bombo_backup:INSTANCE' in snapshot.tags:
if ((snapshot.tags.get('bombo_backup:INSTANCE')
== kInstanceId)
or (str(kInstanceId).lower() == "all")):
#Only purge specified instance snapshot OR all of the bombo snapshots
start_time = datetime.strptime(
snapshot.start_time, '%Y-%m-%dT%H:%M:%S.000Z')
if kKeepHistoricals:
# We want to keep a monthly backup from the 1st of the month until the maximum age as per configured by the customer
if (start_time < delete_time
and start_time.day <> 1):
# If the backup is older than retention period, but it wasn't created on the 1st of the month, delete it
print(
'Deleting {id}'.format(id=snapshot.id)
) + " made on " + str(
snapshot.tags.get('bombo_backup:DATE')
) + " attached to " + str(
snapshot.tags.get('bombo_backup:INSTANCE')
) + " mounted on " + str(
snapshot.tags.get('bombo_backup:DEVICE'))
deletion_counter = deletion_counter + 1
size_counter = size_counter + snapshot.volume_size
try:
snapshot.delete(dry_run=False)
except:
FailedDeleteSnapshot.append(snapshot.id)
elif start_time.day == 1 and start_time < historical_age:
# Else if the snapshot was on the 1st of the month AND it is older than the historical retention period it can be deleted.
print('Deleting monthly {id}'.format(
id=snapshot.id
)) + " made on " + str(
snapshot.tags.get('bombo_backup:DATE')
) + " attached to " + str(
snapshot.tags.get('bombo_backup:INSTANCE')
) + " mounted on " + str(
snapshot.tags.get('bombo_backup:DEVICE'))
deletion_counter = deletion_counter + 1
size_counter = size_counter + snapshot.volume_size
try:
snapshot.delete(dry_run=False)
except:
FailedDeleteSnapshot.append(snapshot.id)
else:
# If the backup is older than retention period, delete it as we are not interested in keeping monthlies
if start_time < delete_time:
print(
'Deleting {id}'.format(id=snapshot.id)
) + " made on " + str(
snapshot.tags.get('bombo_backup:DATE')
) + " attached to " + str(
snapshot.tags.get('bombo_backup:INSTANCE')
) + " mounted on " + str(
snapshot.tags.get('bombo_backup:DEVICE'))
deletion_counter = deletion_counter + 1
size_counter = size_counter + snapshot.volume_size
try:
snapshot.delete(dry_run=False)
except:
FailedDeleteSnapshot.append(snapshot.id)
print 'Deleted {number} snapshots totalling {size} GB'.format(
number=deletion_counter, size=size_counter)
self.printMsg("", "Hopefully everything went well......")
#
# Print any snapshots that failed
if FailedVolumesSnapshot:
self.printMsg("", "Unfortunately, things did not go so well......")
for FailedVolume in FailedVolumesSnapshot:
self.printMsg(
"", "ERROR trying to create snapshot for instance " +
FailedVolume[0].tags.get('bombo_backup:INSTANCE') +
" , Device " +
FailedVolume[0].tags.get('bombo_backup:DEVICE'))
print("")
for FailedDelete in FailedDeleteSnapshot:
self.printMsg(
"", "ERROR trying to delete snapshot " + FailedDelete)
def __getObjCustomer(self, kCustomerId):
self.CustomerId = kCustomerId
self.Customer = clsCustomer(kCustomerId)
def CopyInstance(self,
kCustomerId,
kRegion,
kInstanceId,
kSubnet,
kKeepInstanceOn=False):
from datetime import datetime
import sys, time
import boto.ec2
import boto.vpc
self.showInitialMsg()
if kKeepInstanceOn:
self.printMsg(
"",
"###> You choose to copy the instance without switching it off, please check when done if it was completed successfully!"
)
ObjCustomer = clsCustomer(kCustomerId)
self.printMsg("", "Connecting ...")
self.__awsConnection = boto.ec2.connect_to_region(
kRegion,
aws_access_key_id=ObjCustomer.Access_key,
aws_secret_access_key=ObjCustomer.Secret_key)
self.printMsg("", "---> " + "Connected")
self.printMsg("", "Getting instance details...")
reservations = self.__awsConnection.get_all_instances(
instance_ids=[kInstanceId])
instance = reservations[0].instances[0]
self.printMsg("", "---> " + instance.id)
self.printMsg("", "Getting subnet details...")
self.__awsVpcConnection = boto.vpc.connect_to_region(
kRegion,
aws_access_key_id=ObjCustomer.Access_key,
aws_secret_access_key=ObjCustomer.Secret_key)
subnets = self.__awsVpcConnection.get_all_subnets(subnet_ids=[kSubnet])
old_vpc_id = subnets[0].vpc_id
old_azone = subnets[0].availability_zone
self.printMsg(
"", "---> Found " + kSubnet + " linked to " + old_vpc_id + " in " +
old_azone)
if kKeepInstanceOn == False:
self.printMsg("", "Stopping the instance...")
instances_to_stop = self.__awsConnection.stop_instances(
instance_ids=kInstanceId)
counter = 0
spinner = self.spinning_cursor()
while counter != len(instances_to_stop):
sys.stdout.write(spinner.next())
sys.stdout.flush()
sys.stdout.write('\b')
for instance in instances_to_stop:
if instance.update() == "stopped":
counter += 1
time.sleep(2)
self.printMsg("", "---> Done")
self.printMsg("", "Getting volumes...")
vols = self.__awsConnection.get_all_volumes(
filters={'attachment.instance-id': instance.id})
self.printMsg("", "---> " + str(len(vols)) + " Volumes found")
self.printMsg("", "Tagging instance...")
self.__awsConnection.create_tags(
[instance.id], {
"bombo_moving":
kSubnet + " - " +
datetime.today().strftime('%d-%m-%Y %H:%M:%S')
})
self.printMsg(
"", "---> " + kSubnet + " - " +
datetime.today().strftime('%d-%m-%Y %H:%M:%S'))
#self.printMsg ("","Copying tags...")
#TagsList = []
#TagsList = setTagsToInstance([instance.id])
#self.printMsg ("","Finished copying tags")
VolumesSnapshotMatchList = []
for vol in vols:
self.printMsg("", "Tagging volume " + vol.id + "...")
if instance.tags.get('Name'):
vol.add_tag(
"Name",
"### Copy of :" + instance.tags.get('Name') + " ###")
else:
vol.add_tag("Name", "### Copied ###")
vol.add_tag("bombo_moving:INSTANCE", instance.id)
vol.add_tag("bombo_moving:STATUS", vol.attach_data.status)
vol.add_tag("bombo_moving:DEVICE", vol.attach_data.device)
self.printMsg("", "Snapshot volume " + vol.id + "...")
snapshot = self.__awsConnection.create_snapshot(
vol.id, "INSTANCE: " + instance.id + " - DEVICE:" +
vol.attach_data.device)
VolumesSnapshotMatchList.append([vol, snapshot])
self.printMsg(
"", "---> Done with [" + str(vol.id) + "] => " + "INSTANCE: " +
instance.id + " - DEVICE:" + vol.attach_data.device)
self.printMsg("", "Waiting for the snapshots to be ready...")
counter = 0
progress = 0
singleValue = ""
spinner = self.spinning_cursor()
while progress < (100 * len(VolumesSnapshotMatchList)):
progress = 0
for VolumesSnapshotMatch in VolumesSnapshotMatchList:
singleValue = VolumesSnapshotMatch[1].update()
if len(singleValue) == 0:
singleValue = "0%"
progress = progress + int(singleValue[:-1])
time.sleep(2)
sys.stdout.flush()
sys.stdout.write('\r')
sys.stdout.write(spinner.next() + " Waiting for " +
str(len(VolumesSnapshotMatchList)) +
" snapshots => Progress: " +
str(progress / len(VolumesSnapshotMatchList)) +
"%")
sys.stdout.write('\n')
self.printMsg("", "---> Snapshots ready...")
NewInstanceVolumeList = []
self.printMsg("", "Creating volumes from snapshot...")
for VolumesSnapshotMatch in VolumesSnapshotMatchList:
new_vol_tmp = self.__awsConnection.create_volume(
size=VolumesSnapshotMatch[0].size,
volume_type=VolumesSnapshotMatch[0].type,
snapshot=VolumesSnapshotMatch[1],
zone=old_azone)
new_vol_tmp.add_tag("bombo_moving:DATE",
datetime.today().strftime('%d-%m-%Y %H:%M:%S'))
new_vol_tmp.add_tag(
"bombo_moving:SOURCE", kInstanceId + " -> " +
VolumesSnapshotMatch[0].attach_data.device)
NewInstanceVolumeList.append(
[VolumesSnapshotMatch[0].attach_data.device, new_vol_tmp])
self.printMsg(
"", "---> [" + VolumesSnapshotMatch[0].attach_data.device +
"] - " + VolumesSnapshotMatch[0].type + ", " +
str(VolumesSnapshotMatch[0].size) + " GB created")
self.printMsg("", "---> Volumes ready...")
self.printMsg("",
"Check AMI availability " + instance.image_id + "...")
ObjInstance = clsInstance(instance.id, ObjCustomer, instance)
if ObjInstance.checkAmiAvailility(self.__awsConnection):
self.printMsg("", "---> AMI Available")
ObjInstance.Ami = instance.image_id
else:
self.printMsg(
"", "###> AMI NOT Available, searching for a similar AMI [" +
instance.architecture + " - " + instance.virtualization_type +
" - " + instance.hypervisor + " - " +
instance.root_device_type + "]")
ObjInstance.Ami = self.searchSimilarAMI(self.__awsConnection,
instance)
self.printMsg("", "Launching new instance on " + kSubnet + "...")
reservation = self.__awsConnection.run_instances(
image_id=ObjInstance.Ami,
instance_type=instance.instance_type,
key_name=instance.key_name,
#private_ip_address = "10.0.34.100",
#security_group_ids = kSingleLaunch.SecGroups,
subnet_id=kSubnet,
dry_run=False,
)
counter = 0
spinner = self.spinning_cursor()
while counter != len(reservation.instances):
sys.stdout.write(spinner.next())
sys.stdout.flush()
sys.stdout.write('\b')
for instance in reservation.instances:
if instance.update() != "pending":
counter += 1
time.sleep(
2
) #Must wait even when the instance is ready, otherwise we got instance not found by AWS
self.printMsg(
"",
"---> [" + reservation.instances[0].id + "] => Instance launched")
self.printMsg("", "Tagging new instance...")
self.__awsConnection.create_tags(
[instance.id], {"bombo_moving:SOURCE_INSTANCE": kInstanceId})
self.__awsConnection.create_tags([instance.id], {
"bombo_moving:DATE":
datetime.today().strftime('%d-%m-%Y %H:%M:%S')
})
# self.printMsg ("","Extracting tags from source instance...")
# TagsList = self.getTagsFromInstance(kInstanceId)
# self.printMsg ("","Applying tags to the new instance...")
# self.setTagsToInstance(TagsList, instance.id)
self.printMsg("", "---> New instance tagged...")
self.printMsg("", "Stopping NEW instance...")
self.__awsConnection.stop_instances(
instance_ids=reservation.instances[0].id, force=True)
counter = 0
spinner = self.spinning_cursor()
while counter != len(reservation.instances):
sys.stdout.write(spinner.next())
sys.stdout.flush()
sys.stdout.write('\b')
for instance in reservation.instances:
if instance.update() == "stopped":
counter += 1
time.sleep(2)
self.printMsg("", "---> Done")
self.printMsg("", "Getting volumes list of the new instance...")
new_instance_vols = self.__awsConnection.get_all_volumes(
filters={'attachment.instance-id': reservation.instances[0].id})
self.printMsg(
"", "---> " + str(len(vols)) + " volumes found, deleting all..")
for new_instance_vol in new_instance_vols:
new_instance_vol.detach(force=True)
self.printMsg("",
"---> " + str(new_instance_vol.id) + " detached....")
new_instance_vol.delete
self.printMsg(
"", "---> " + str(new_instance_vol.id) + " and now deleted!")
counter = 0
spinner = self.spinning_cursor()
while counter < len(new_instance_vols):
sys.stdout.write(spinner.next())
sys.stdout.flush()
sys.stdout.write('\b')
for vol_check_detach in new_instance_vols:
if vol_check_detach.update() == "available":
counter += 1
time.sleep(2)
self.printMsg("", "Attaching volumes to the new instance...")
for NewInstanceVolume in NewInstanceVolumeList:
self.__awsConnection.attach_volume(
instance_id=reservation.instances[0].id,
volume_id=NewInstanceVolume[1].id,
device=NewInstanceVolume[0])
self.printMsg(
"", "---> " + NewInstanceVolume[0] + " attached to " +
str(reservation.instances[0].id))
self.printMsg("", "Starting the NEW instance...")
self.__awsConnection.start_instances(
instance_ids=reservation.instances[0].id)
self.printMsg("", "---> Done")
self.printMsg("", "Hopefully everything went well......")
