def main(argv=None):
"""Parse command line options and dump a datacenter to snapshots and file."""
if argv is None:
argv = sys.argv
else:
sys.argv.extend(argv)
program_name = os.path.basename(sys.argv[0])
program_version = "v%s" % __version__
program_build_date = str(__updated__)
program_version_message = "%%(prog)s %s (%s)" % (program_version, program_build_date)
program_shortdesc = __import__("__main__").__doc__.split("\n")[1]
program_license = """%s
Created by J. Buchhammer on %s.
Copyright 2016 ProfitBricks GmbH. All rights reserved.
Licensed under the Apache License 2.0
http://www.apache.org/licenses/LICENSE-2.0
Distributed on an "AS IS" basis without warranties
or conditions of any kind, either express or implied.
USAGE
""" % (
program_shortdesc,
str(__date__),
)
try:
# Setup argument parser
parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter)
parser.add_argument("-u", "--user", dest="user", help="the login name")
parser.add_argument("-p", "--password", dest="password", help="the login password")
parser.add_argument("-L", "--Login", dest="loginfile", default=None, help="the login file to use")
parser.add_argument(
"-d", "--datacenterid", dest="dc_id", required=True, default=None, help="datacenter ID of the server"
)
parser.add_argument(
"-o",
"--outfile",
dest="outfile",
default="dc-def_" + datetime.now().strftime("%Y-%m-%d_%H%M%S"),
help="the output file name",
)
parser.add_argument(
"-S", "--Stopalways", dest="stopalways", action="store_true", help="power off even when VM is running"
)
parser.add_argument(
"-v",
"--verbose",
dest="verbose",
action="count",
default=0,
help="set verbosity level [default: %(default)s]",
)
parser.add_argument("-V", "--version", action="version", version=program_version_message)
# Process arguments
args = parser.parse_args()
global verbose
verbose = args.verbose
if verbose > 0:
print("Verbose mode on")
print("start {} with args {}".format(program_name, str(args)))
outfile = args.outfile
if outfile.endswith(".json"):
outfile = os.path.splitext(outfile)
print("Using output file base name '{}'".format(outfile))
(user, password) = getLogin(args.loginfile, args.user, args.password)
if user is None or password is None:
raise ValueError("user or password resolved to None")
pbclient = ProfitBricksService(user, password)
dc_id = args.dc_id
# first get all server's VM and OS state to see if we can start
srv_info = getServerInfo(pbclient, dc_id)
srvon = 0
for server in srv_info:
if server["vmstate"] != "SHUTOFF":
print("VM {} is in state {}, but should be SHUTOFF".format(server["name"], server["vmstate"]))
srvon += 1
# end for(srv_info)
if srvon > 0 and not args.stopalways:
print("shutdown running OS before trying again")
return 1
# now power off all VMs before starting the snapshots
for server in srv_info:
controlServerState(pbclient, dc_id, server["id"], action="POWEROFF")
# now let's go
dcdef = pbclient.get_datacenter(dc_id, 5)
print("starting dump of datacenter {}".format(dcdef["properties"]["name"]))
dcdef_file = outfile + "_source.json"
print("write source dc to {}".format(dcdef_file))
write_dc_definition(pbclient, dcdef, dcdef_file)
print("get existing Snapshots")
# first get existing snapshots
known_snapshots = dict()
snapshots = pbclient.list_snapshots()
for snap in snapshots["items"]:
print("SNAP : {}".format(json.dumps(snap)))
known_snapshots[snap["properties"]["name"]] = snap["id"]
print("create Snapshots, this may take a while ..")
# we do NOT consider dangling volumes, only server-attached ones
vol_snapshots = dict() # map volume id==snapshot name snapshot id
for server in dcdef["entities"]["servers"]["items"]:
print("- server {}".format(server["properties"]["name"]))
if "volumes" not in server["entities"]:
print(" server {} has no volumes".format(server["properties"]["name"]))
continue
# The volumes are attached by order of creation
# Thus we must sort them to keep the order in the clone
print("setting volume order by deviceNumber")
volumes = server["entities"]["volumes"]["items"]
new_order = sorted(volumes, key=lambda vol: vol["properties"]["deviceNumber"])
server["entities"]["volumes"]["items"] = new_order
for volume in server["entities"]["volumes"]["items"]:
vol_id = volume["id"] # this will be the name too
if vol_id in known_snapshots:
print("use existing snapshot {} of volume {}".format(vol_id, volume["properties"]["name"]))
vol_snapshots[vol_id] = known_snapshots[vol_id]
else:
print("taking snapshot {} of volume {}".format(vol_id, volume["properties"]["name"]))
response = pbclient.create_snapshot(dc_id, vol_id, vol_id, "auto-created by pb_snapshotDatacenter")
# response has no request id, need to check metadata state (BUSY, AVAILABLE..)
vol_snapshots[vol_id] = response["id"]
print("snapshot in progress: {}".format(str(response)))
# end for(volume)
# end for(server)
print("Waiting for snapshots to complete")
snapdone = dict()
while len(snapdone) != len(vol_snapshots):
sleep(10)
for snap_id in vol_snapshots.values():
print("looking for {}".format(snap_id))
if snap_id in snapdone:
continue
snapshot = pbclient.get_snapshot(snap_id)
print("snapshot {} is in state {}".format(snap_id, snapshot["metadata"]["state"]))
if snapshot["metadata"]["state"] == "AVAILABLE":
snapdone[snap_id] = snapshot["metadata"]["state"]
# end for(vol_snapshots)
# end while(snapdone)
# now replace the volumes image IDs
print("setting snapshot id to volumes")
for server in dcdef["entities"]["servers"]["items"]:
print("- server {}".format(server["properties"]["name"]))
if "volumes" not in server["entities"]:
print(" server {} has no volumes".format(server["properties"]["name"]))
continue
for volume in server["entities"]["volumes"]["items"]:
vol_id = volume["id"] # this will be the name too
volume["properties"]["image"] = vol_snapshots[vol_id]
# end for(volume)
# end for(server)
# As it came out, the LAN id is rearranged by order of creation
# Thus we must sort the LANs to keep the order in the clone
print("setting LAN order by id")
lans = dcdef["entities"]["lans"]["items"]
new_order = sorted(lans, key=lambda lan: lan["id"])
dcdef["entities"]["lans"]["items"] = new_order
# now sort unordered NICs by MAC and save the dcdef
# reason is, that NICs seem to be ordered by MAC, but API response
# doesn't guarantee the order, which we need for re-creation
print("setting NIC order by MAC")
for server in dcdef["entities"]["servers"]["items"]:
print("- server {}".format(server["properties"]["name"]))
if "nics" not in server["entities"]:
print(" server {} has no nics".format(server["properties"]["name"]))
continue
nics = server["entities"]["nics"]["items"]
# print("NICs before {}".format(json.dumps(nics)))
new_order = sorted(nics, key=lambda nic: nic["properties"]["mac"])
# print("NICs after {}".format(json.dumps(new_order)))
server["entities"]["nics"]["items"] = new_order
# end for(server)
dcdef_file = outfile + ".json"
print("write snapshot dc to {}".format(dcdef_file))
write_dc_definition(pbclient, dcdef, dcdef_file)
return 0
except KeyboardInterrupt:
### handle keyboard interrupt ###
return 0
except Exception:
traceback.print_exc()
sys.stderr.write("\n" + program_name + ": for help use --help\n")
return 2
class TestVolume(unittest.TestCase):
@classmethod
def setUpClass(self):
self.resource = resource()
self.client = ProfitBricksService(
username=configuration.USERNAME,
password=configuration.PASSWORD,
headers=configuration.HEADERS)
# Create test datacenter
self.datacenter = self.client.create_datacenter(
datacenter=Datacenter(**self.resource['datacenter']))
wait_for_completion(self.client, self.datacenter, 'create_datacenter')
# Create test volume
self.volume = self.client.create_volume(
datacenter_id=self.datacenter['id'],
volume=Volume(**self.resource['volume']))
wait_for_completion(self.client, self.volume, 'create_volume')
# Create snapshot1
self.snapshot1 = self.client.create_snapshot(
datacenter_id=self.datacenter['id'],
volume_id=self.volume['id'],
name=self.resource['snapshot']['name'],
description=self.resource['snapshot']['description'])
wait_for_completion(self.client, self.snapshot1, 'create_snapshot1',
wait_timeout=600)
# Create snapshot2 (used in delete test)
self.snapshot2 = self.client.create_snapshot(
datacenter_id=self.datacenter['id'],
volume_id=self.volume['id'],
name=self.resource['snapshot']['name'],
description=self.resource['snapshot']['description'])
wait_for_completion(self.client, self.snapshot2, 'create_snapshop2',
wait_timeout=600)
self.image = find_image(self.client, configuration.IMAGE_NAME)
@classmethod
def tearDownClass(self):
self.client.remove_snapshot(snapshot_id=self.snapshot1['id'])
self.client.delete_datacenter(datacenter_id=self.datacenter['id'])
def test_list_volumes(self):
volumes = self.client.list_volumes(
datacenter_id=self.datacenter['id'])
self.assertGreater(len(volumes), 0)
assertRegex(self, volumes['items'][0]['id'], self.resource['uuid_match'])
self.assertEqual(volumes['items'][0]['type'], 'volume')
self.assertEqual(volumes['items'][0]['properties']['name'], self.resource['volume']['name'])
self.assertEqual(volumes['items'][0]['properties']['size'], self.resource['volume']['size'])
self.assertEqual(volumes['items'][0]['properties']['licenceType'], self.resource['volume']['licence_type'])
self.assertEqual(volumes['items'][0]['properties']['type'], self.resource['volume']['type'])
self.assertFalse(volumes['items'][0]['properties']['cpuHotPlug'])
self.assertFalse(volumes['items'][0]['properties']['cpuHotUnplug'])
self.assertFalse(volumes['items'][0]['properties']['ramHotPlug'])
self.assertFalse(volumes['items'][0]['properties']['ramHotUnplug'])
self.assertFalse(volumes['items'][0]['properties']['nicHotPlug'])
self.assertFalse(volumes['items'][0]['properties']['nicHotUnplug'])
self.assertFalse(volumes['items'][0]['properties']['discVirtioHotPlug'])
self.assertFalse(volumes['items'][0]['properties']['discVirtioHotUnplug'])
self.assertFalse(volumes['items'][0]['properties']['discScsiHotPlug'])
self.assertFalse(volumes['items'][0]['properties']['discScsiHotUnplug'])
self.assertIsNone(volumes['items'][0]['properties']['bus'])
def test_get_volume(self):
volume = self.client.get_volume(
datacenter_id=self.datacenter['id'],
volume_id=self.volume['id'])
self.assertEqual(volume['id'], self.volume['id'])
self.assertEqual(volume['type'], 'volume')
self.assertEqual(volume['properties']['name'], self.resource['volume']['name'])
self.assertEqual(volume['properties']['size'], self.resource['volume']['size'])
self.assertEqual(volume['properties']['licenceType'], self.resource['volume']['licence_type'])
self.assertEqual(volume['properties']['type'], self.resource['volume']['type'])
self.assertFalse(volume['properties']['cpuHotPlug'])
self.assertFalse(volume['properties']['cpuHotUnplug'])
self.assertFalse(volume['properties']['ramHotPlug'])
self.assertFalse(volume['properties']['ramHotUnplug'])
self.assertFalse(volume['properties']['nicHotPlug'])
self.assertFalse(volume['properties']['nicHotUnplug'])
self.assertFalse(volume['properties']['discVirtioHotPlug'])
self.assertFalse(volume['properties']['discVirtioHotUnplug'])
self.assertFalse(volume['properties']['discScsiHotPlug'])
self.assertFalse(volume['properties']['discScsiHotUnplug'])
self.assertIsNone(volume['properties']['bus'])
def test_delete_volume(self):
volume = self.client.create_volume(
datacenter_id=self.datacenter['id'],
volume=Volume(**self.resource['volume']))
wait_for_completion(self.client, volume, 'create_volume')
volume = self.client.delete_volume(
datacenter_id=self.datacenter['id'],
volume_id=volume['id'])
self.assertTrue(volume)
def test_update_volume(self):
volume = self.client.update_volume(
datacenter_id=self.datacenter['id'],
volume_id=self.volume['id'],
size=6,
name=self.resource['volume']['name'] + ' RENAME')
wait_for_completion(self.client, volume, 'update_volume')
volume = self.client.get_volume(
datacenter_id=self.datacenter['id'],
volume_id=self.volume['id'])
self.assertEqual(volume['id'], self.volume['id'])
self.assertEqual(volume['properties']['name'], self.resource['volume']['name'] + ' RENAME')
self.assertEqual(volume['properties']['size'], 6)
def test_create_volume(self):
# Use volume created during volume test setup.
self.assertEqual(self.volume['properties']['name'], self.resource['volume']['name'])
self.assertEqual(self.volume['properties']['bus'], self.resource['volume']['bus'])
self.assertEqual(self.volume['properties']['type'], self.resource['volume']['type'])
self.assertEqual(self.volume['properties']['size'], self.resource['volume']['size'])
self.assertEqual(self.volume['properties']['licenceType'], self.resource['volume']['licence_type'])
self.assertFalse(self.volume['properties']['cpuHotPlug'])
self.assertFalse(self.volume['properties']['cpuHotUnplug'])
self.assertFalse(self.volume['properties']['ramHotPlug'])
self.assertFalse(self.volume['properties']['ramHotUnplug'])
self.assertFalse(self.volume['properties']['nicHotPlug'])
self.assertFalse(self.volume['properties']['nicHotUnplug'])
self.assertFalse(self.volume['properties']['discVirtioHotPlug'])
self.assertFalse(self.volume['properties']['discVirtioHotUnplug'])
self.assertFalse(self.volume['properties']['discScsiHotPlug'])
self.assertFalse(self.volume['properties']['discScsiHotUnplug'])
def test_create_snapshot(self):
# Use snapshot created during volume test setup.
assertRegex(self, self.snapshot1['id'], self.resource['uuid_match'])
self.assertEqual(self.snapshot1['type'], 'snapshot')
self.assertEqual(self.snapshot1['properties']['name'], self.resource['snapshot']['name'])
self.assertEqual(self.snapshot1['properties']['description'], self.resource['snapshot']['description'])
self.assertEqual(self.snapshot1['properties']['location'], configuration.LOCATION)
self.assertFalse(self.snapshot1['properties']['cpuHotPlug'])
self.assertFalse(self.snapshot1['properties']['cpuHotUnplug'])
self.assertFalse(self.snapshot1['properties']['ramHotPlug'])
self.assertFalse(self.snapshot1['properties']['ramHotUnplug'])
self.assertFalse(self.snapshot1['properties']['nicHotPlug'])
self.assertFalse(self.snapshot1['properties']['nicHotUnplug'])
self.assertFalse(self.snapshot1['properties']['discVirtioHotPlug'])
self.assertFalse(self.snapshot1['properties']['discVirtioHotUnplug'])
self.assertFalse(self.snapshot1['properties']['discScsiHotPlug'])
self.assertFalse(self.snapshot1['properties']['discScsiHotUnplug'])
self.assertIsNone(self.snapshot1['properties']['size'])
self.assertIsNone(self.snapshot1['properties']['licenceType'])
def test_restore_snapshot(self):
response = self.client.restore_snapshot(
datacenter_id=self.datacenter['id'],
volume_id=self.volume['id'],
snapshot_id=self.snapshot1['id'])
self.assertTrue(response)
def test_remove_snapshot(self):
volume = self.client.remove_snapshot(snapshot_id=self.snapshot2['id'])
self.assertTrue(volume)
def test_create_volume_failure(self):
with self.assertRaises(Exception) as context:
self.client.create_volume(
datacenter_id=self.datacenter['id'],
volume=Volume(image=self.image['id'],
**self.resource['volume_failure']))
exception = ('(422, u\'[(root).properties.image] Passwords/SSH Keys '
'are mandatory for public ProfitBricks Images.\')')
self.assertIn(exception, str(context.exception))
# timestamp for creating and deleting
now = datetime.now()
# create new snapshots from list of volumes in datacenter
volumes = pb.list_volumes(datacenter_id=datacenter_id)
for volume_item in volumes['items']:
volume_id = volume_item['id']
volume = pb.get_volume(datacenter_id, volume_id)
volume_name = volume['properties']['name']
snapshot_name = "{}-{}-{}".format(snapshot_prefix, now.strftime("%Y%m%d"), volume_name)
print("creating snapshot: {}".format(snapshot_name))
pb.create_snapshot(datacenter_id, volume_id, snapshot_name)
sleep(sleep_seconds)
# delete old snapshots
snapshots = pb.list_snapshots()
for snapshot_item in snapshots['items']:
snapshot_id = snapshot_item['id']
snapshot = pb.get_snapshot(snapshot_id)
snapshot_name = snapshot['properties']['name']
if not snapshot_name.startswith(snapshot_prefix):
continue
snapshot_created = snapshot['metadata']['createdDate']
snapshot_date = datetime.strptime(snapshot_created, "%Y-%m-%dT%H:%M:%SZ")
# timestamp for creating and deleting
now = datetime.now()
# create new snapshots from list of volumes in datacenter
volumes = pb.list_volumes(datacenter_id=datacenter_id)
for volume_item in volumes['items']:
volume_id = volume_item['id']
volume = pb.get_volume(datacenter_id, volume_id)
volume_name = volume['properties']['name']
snapshot_name = "{}-{}-{}".format(snapshot_prefix, now.strftime("%Y%m%d"),
volume_name)
print("creating snapshot: {}".format(snapshot_name))
pb.create_snapshot(datacenter_id, volume_id, snapshot_name)
sleep(sleep_seconds)
# delete old snapshots
snapshots = pb.list_snapshots()
for snapshot_item in snapshots['items']:
snapshot_id = snapshot_item['id']
snapshot = pb.get_snapshot(snapshot_id)
snapshot_name = snapshot['properties']['name']
if not snapshot_name.startswith(snapshot_prefix):
continue
snapshot_created = snapshot['metadata']['createdDate']
snapshot_date = datetime.strptime(snapshot_created, "%Y-%m-%dT%H:%M:%SZ")
datacenter_id=datacenter_id, volume=i)
"""Create snapshot
"""
from profitbricks.client import ProfitBricksService
datacenter_id = '700e1cab-99b2-4c30-ba8c-1d273ddba022'
volume_id = '700e1cab-99b2-4c30-ba8c-1d273ddba025'
client = ProfitBricksService(
username='******', password='******')
volume = client.create_snapshot(
datacenter_id=datacenter_id,
volume_id=volume_id,
snapshot_name='<URLENCODED_SNAPSHOT_NAME>',
snapshot_description='<URLENCODED_SNAPSHOT_DESCRIPTION>')
"""Restore Snapshot
"""
from profitbricks.client import ProfitBricksService
datacenter_id = '700e1cab-99b2-4c30-ba8c-1d273ddba022'
volume_id = '700e1cab-99b2-4c30-ba8c-1d273ddba025'
snapshot_id = '7df81087-5835-41c6-a10b-3e098593bba4'
client = ProfitBricksService(
username='******', password='******')
class TestVolume(unittest.TestCase):
@classmethod
def setUpClass(self):
self.resource = resource()
self.client = ProfitBricksService(
username=configuration.USERNAME,
password=configuration.PASSWORD,
headers=configuration.HEADERS)
# Create test datacenter
self.datacenter = self.client.create_datacenter(
datacenter=Datacenter(**self.resource['datacenter']))
self.client.wait_for_completion(self.datacenter)
self.image = find_image(self.client, configuration.IMAGE_NAME)
# Create test volume
vol = Volume(**self.resource['volume2'])
vol.image = self.image['id']
self.volume = self.client.create_volume(
datacenter_id=self.datacenter['id'],
volume=vol)
self.client.wait_for_completion(self.volume)
# Create snapshot1
self.snapshot1 = self.client.create_snapshot(
datacenter_id=self.datacenter['id'],
volume_id=self.volume['id'],
name=self.resource['snapshot']['name'],
description=self.resource['snapshot']['description'])
self.client.wait_for_completion(self.snapshot1, timeout=600)
@classmethod
def tearDownClass(self):
self.client.delete_datacenter(datacenter_id=self.datacenter['id'])
def test_list_volumes(self):
volumes = self.client.list_volumes(
datacenter_id=self.datacenter['id'])
self.assertGreater(len(volumes), 0)
assertRegex(self, volumes['items'][0]['id'], self.resource['uuid_match'])
self.assertEqual(volumes['items'][0]['type'], 'volume')
self.assertEqual(volumes['items'][0]['properties']['name'],
self.resource['volume2']['name'])
self.assertEqual(volumes['items'][0]['properties']['size'],
self.resource['volume2']['size'])
self.assertEqual(volumes['items'][0]['properties']['type'],
self.resource['volume2']['disk_type'])
self.assertIsNone(volumes['items'][0]['properties']['bus'])
def test_get_volume(self):
volume = self.client.get_volume(
datacenter_id=self.datacenter['id'],
volume_id=self.volume['id'])
self.assertEqual(volume['id'], self.volume['id'])
self.assertEqual(volume['type'], 'volume')
self.assertEqual(volume['properties']['name'], self.resource['volume2']['name'])
self.assertEqual(volume['properties']['size'], self.resource['volume2']['size'])
self.assertEqual(volume['properties']['licenceType'],
self.image['properties']['licenceType'])
self.assertEqual(volume['properties']['type'], self.resource['volume2']['disk_type'])
self.assertIsNone(volume['properties']['bus'])
self.assertEqual(volume['properties']['availabilityZone'],
self.resource['volume2']['availability_zone'])
def test_delete_volume(self):
volume = self.client.create_volume(
datacenter_id=self.datacenter['id'],
volume=Volume(**self.resource['volume']))
self.client.wait_for_completion(volume)
volume = self.client.delete_volume(
datacenter_id=self.datacenter['id'],
volume_id=volume['id'])
self.assertTrue(volume)
def test_update_volume(self):
volume = self.client.update_volume(
datacenter_id=self.datacenter['id'],
volume_id=self.volume['id'],
size=6,
name=self.resource['volume2']['name'] + ' - RENAME')
self.client.wait_for_completion(volume)
volume = self.client.get_volume(
datacenter_id=self.datacenter['id'],
volume_id=self.volume['id'])
self.assertEqual(volume['id'], self.volume['id'])
self.assertEqual(volume['properties']['name'],
self.resource['volume2']['name'] + ' - RENAME')
self.assertEqual(volume['properties']['size'], 6)
def test_create_volume(self):
# Use volume created during volume test setup.
assertRegex(self, self.volume['id'], self.resource['uuid_match'])
self.assertEqual(self.volume['properties']['name'], self.resource['volume2']['name'])
self.assertEqual(self.volume['properties']['bus'], self.resource['volume2']['bus'])
self.assertEqual(self.volume['properties']['type'], self.resource['volume2']['disk_type'])
self.assertEqual(self.volume['properties']['size'], self.resource['volume2']['size'])
self.assertEqual(self.volume['properties']['sshKeys'],
self.resource['volume2']['ssh_keys'])
self.assertEqual(self.volume['properties']['availabilityZone'],
self.resource['volume2']['availability_zone'])
def test_create_snapshot(self):
# Use snapshot created during volume test setup.
self.assertEqual(self.snapshot1['type'], 'snapshot')
self.assertEqual(self.snapshot1['properties']['name'], self.resource['snapshot']['name'])
self.assertEqual(self.snapshot1['properties']['description'],
self.resource['snapshot']['description'])
self.assertEqual(self.snapshot1['properties']['location'], configuration.LOCATION)
self.assertIsNone(self.snapshot1['properties']['size'])
self.assertIsNone(self.snapshot1['properties']['licenceType'])
def test_restore_snapshot(self):
response = self.client.restore_snapshot(
datacenter_id=self.datacenter['id'],
volume_id=self.volume['id'],
snapshot_id=self.snapshot1['id'])
self.assertTrue(response)
def test_remove_snapshot(self):
volume = self.client.remove_snapshot(snapshot_id=self.snapshot1['id'])
self.assertTrue(volume)
def test_get_failure(self):
try:
self.client.get_volume(
datacenter_id=self.datacenter['id'],
volume_id='00000000-0000-0000-0000-000000000000')
except PBNotFoundError as e:
self.assertIn(self.resource['not_found_error'], e.content[0]['message'])
def test_create_failure(self):
try:
volume = Volume(name=self.resource['volume2']['name'])
self.client.create_volume(datacenter_id=self.datacenter['id'], volume=volume)
except PBError as e:
self.assertIn(self.resource['missing_attribute_error'] % 'size',
e.content[0]['message'])
print "Server '%s' snapshot '%s' deleted, older than %d retention days (%d days, %d seconds)" % (servername, snapname, config.RETENTION_DAYS, snapage.days, snapage.seconds)
else:
# Recent, keep it
print "Server '%s' snapshot '%s' kept, not older than %d retention days (%d days, %d seconds)" % (servername, snapname, config.RETENTION_DAYS, snapage.days, snapage.seconds)
# For recent backups, check if it's less than min_snap_hours
if snapage < timedelta(hours=config.MIN_SNAP_HOURS):
# Snapshot is recent, don't make a new one
no_recent_snapshot = False
# Does this volume have recent snapshots?
if no_recent_snapshot:
# nothing recent, create one. Following PB default name syntax, e.g. "{volumename}-Snapshot-MM/DD/YYYY"
snapname = volumename + '-Snapshot-' + datetime.utcnow().strftime('%m/%d/%Y')
# Do the actual snapshot creation api call
newsnapshot = client.create_snapshot(datacenter_id=config.DATACENTER_ID,volume_id=volumeid, name=snapname)
print "Created new snapshot '%s' with id '%s'" % (snapname, newsnapshot['id'])
else:
# No need to make a snapshot, we have a recent one
print "Found snapshot less than %d hours old, not making new one" % config.MIN_SNAP_HOURS
else:
# This server is not managed based on config settings
print "Not managing snapshots for %s" % servername
class TestVolume(unittest.TestCase):
def setUp(self):
self.volume = ProfitBricksService(
username='******', password='******')
def test_list_volumes(self):
volumes = self.volume.list_volumes(
datacenter_id=datacenter_id)
self.assertEqual(len(volumes), 4)
self.assertEqual(volumes['items'][0]['id'], volume_id)
self.assertEqual(volumes['items'][0]['properties']['name'], 'my boot volume for server 1')
self.assertEqual(volumes['items'][0]['properties']['size'], 80)
self.assertEqual(volumes['items'][0]['properties']['licenceType'], 'WINDOWS')
self.assertFalse(volumes['items'][0]['properties']['cpuHotPlug'])
self.assertFalse(volumes['items'][0]['properties']['cpuHotUnplug'])
self.assertFalse(volumes['items'][0]['properties']['ramHotPlug'])
self.assertFalse(volumes['items'][0]['properties']['ramHotUnplug'])
self.assertFalse(volumes['items'][0]['properties']['nicHotPlug'])
self.assertFalse(volumes['items'][0]['properties']['nicHotUnplug'])
self.assertFalse(volumes['items'][0]['properties']['discVirtioHotPlug'])
self.assertFalse(volumes['items'][0]['properties']['discVirtioHotUnplug'])
self.assertFalse(volumes['items'][0]['properties']['discScsiHotPlug'])
self.assertFalse(volumes['items'][0]['properties']['discScsiHotUnplug'])
self.assertEqual(volumes['items'][0]['properties']['bus'], 'VIRTIO')
self.assertEqual(volumes['items'][0]['properties']['type'], 'HDD')
def test_get_volume(self):
volume = self.volume.get_volume(
datacenter_id=datacenter_id,
volume_id=volume_id)
self.assertEqual(volume['properties']['name'], 'my boot volume for server 1')
self.assertEqual(volume['properties']['size'], 80)
self.assertEqual(volume['properties']['licenceType'], 'WINDOWS')
self.assertFalse(volume['properties']['cpuHotPlug'])
self.assertFalse(volume['properties']['cpuHotUnplug'])
self.assertFalse(volume['properties']['ramHotPlug'])
self.assertFalse(volume['properties']['ramHotUnplug'])
self.assertFalse(volume['properties']['nicHotPlug'])
self.assertFalse(volume['properties']['nicHotUnplug'])
self.assertFalse(volume['properties']['discVirtioHotPlug'])
self.assertFalse(volume['properties']['discVirtioHotUnplug'])
self.assertFalse(volume['properties']['discScsiHotPlug'])
self.assertFalse(volume['properties']['discScsiHotUnplug'])
self.assertEqual(volume['properties']['bus'], 'VIRTIO')
self.assertEqual(volume['properties']['type'], 'HDD')
def test_delete_volume(self):
volume = self.volume.delete_volume(
datacenter_id=datacenter_id,
volume_id=volume_id)
self.assertTrue(volume)
def test_update_volume(self):
volume = self.volume.update_volume(
datacenter_id=datacenter_id,
volume_id=volume_id,
size=100,
name='Resized storage to 100 GB',
cpu_hot_unplug=True)
self.assertEqual(
volume['properties']['name'], 'Resized storage to 100 GB')
self.assertEqual(volume['properties']['size'], 100)
def test_create_volume(self):
i = Volume(
name='Explicitly created volume',
size=56,
image='<IMAGE/SNAPSHOT-ID>',
bus='VIRTIO')
response = self.volume.create_volume(
datacenter_id=datacenter_id, volume=i)
self.assertEqual(
response['properties']['name'], 'my boot volume for server 1')
self.assertEqual(response['properties']['size'], 80)
self.assertEqual(response['properties']['licenceType'], 'WINDOWS')
self.assertFalse(response['properties']['cpuHotPlug'])
self.assertFalse(response['properties']['cpuHotUnplug'])
self.assertFalse(response['properties']['ramHotPlug'])
self.assertFalse(response['properties']['ramHotUnplug'])
self.assertFalse(response['properties']['nicHotPlug'])
self.assertFalse(response['properties']['nicHotUnplug'])
self.assertFalse(response['properties']['discVirtioHotPlug'])
self.assertFalse(response['properties']['discVirtioHotUnplug'])
self.assertFalse(response['properties']['discScsiHotPlug'])
self.assertFalse(response['properties']['discScsiHotUnplug'])
self.assertEqual(response['properties']['bus'], 'VIRTIO')
self.assertEqual(response['properties']['type'], 'HDD')
def test_create_optional_value(self):
i = Volume(
name='Explicitly created volume',
size=56,
image='<IMAGE/SNAPSHOT-ID>',
bus='VIRTIO',
ram_hot_plug=True,
cpu_hot_unplug=True)
response = self.volume.create_volume(
datacenter_id=datacenter_id, volume=i)
self.assertEqual(
response['properties']['name'], 'my boot volume for server 1')
self.assertEqual(response['properties']['size'], 80)
self.assertEqual(response['properties']['licenceType'], 'WINDOWS')
self.assertFalse(response['properties']['cpuHotPlug'])
self.assertFalse(response['properties']['cpuHotUnplug'])
self.assertFalse(response['properties']['ramHotPlug'])
self.assertFalse(response['properties']['ramHotUnplug'])
self.assertFalse(response['properties']['nicHotPlug'])
self.assertFalse(response['properties']['nicHotUnplug'])
self.assertFalse(response['properties']['discVirtioHotPlug'])
self.assertFalse(response['properties']['discVirtioHotUnplug'])
self.assertFalse(response['properties']['discScsiHotPlug'])
self.assertFalse(response['properties']['discScsiHotUnplug'])
self.assertEqual(response['properties']['bus'], 'VIRTIO')
self.assertEqual(response['properties']['type'], 'HDD')
def test_create_snapshot(self):
volume = self.volume.create_snapshot(
datacenter_id=datacenter_id,
volume_id=volume_id,
name='<URLENCODED_SNAPSHOT_NAME>',
description='<URLENCODED_SNAPSHOT_DESCRIPTION>')
self.assertEqual(volume['id'], snapshot_id)
self.assertEqual(
volume['properties']['name'],
'Snapshot of storage X on 12.12.12 12:12:12 - updated')
self.assertEqual(volume['properties']['description'],
'description of a snapshot - updated')
self.assertEqual(volume['properties']['location'], 'de/fkb')
self.assertEqual(volume['properties']['size'], 28)
self.assertEqual(volume['properties']['licenceType'], 'WINDOWS')
self.assertFalse(volume['properties']['cpuHotPlug'])
self.assertFalse(volume['properties']['cpuHotUnplug'])
self.assertFalse(volume['properties']['ramHotPlug'])
self.assertFalse(volume['properties']['ramHotUnplug'])
self.assertFalse(volume['properties']['nicHotPlug'])
self.assertFalse(volume['properties']['nicHotUnplug'])
self.assertFalse(volume['properties']['discVirtioHotPlug'])
self.assertFalse(volume['properties']['discVirtioHotUnplug'])
self.assertFalse(volume['properties']['discScsiHotPlug'])
self.assertFalse(volume['properties']['discScsiHotUnplug'])
def test_restore_snapshot(self):
response = self.volume.restore_snapshot(
datacenter_id=datacenter_id,
volume_id=volume_id,
snapshot_id=snapshot_id)
self.assertTrue(response)
def test_remove_snapshot(self):
volume = self.volume.remove_snapshot(snapshot_id=snapshot_id)
self.assertTrue(volume)
def main(argv=None):
'''Parse command line options and dump a datacenter to snapshots and file.'''
if argv is None:
argv = sys.argv
else:
sys.argv.extend(argv)
program_name = os.path.basename(sys.argv[0])
program_version = "v%s" % __version__
program_build_date = str(__updated__)
program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
program_shortdesc = __import__('__main__').__doc__.split("\n")[1]
program_license = '''%s
Created by J. Buchhammer on %s.
Copyright 2016 ProfitBricks GmbH. All rights reserved.
Licensed under the Apache License 2.0
http://www.apache.org/licenses/LICENSE-2.0
Distributed on an "AS IS" basis without warranties
or conditions of any kind, either express or implied.
USAGE
''' % (program_shortdesc, str(__date__))
try:
# Setup argument parser
parser = ArgumentParser(description=program_license,
formatter_class=RawDescriptionHelpFormatter)
parser.add_argument('-u', '--user', dest='user', help='the login name')
parser.add_argument('-p', '--password', dest='password',
help='the login password')
parser.add_argument('-L', '--Login', dest='loginfile', default=None,
help='the login file to use')
parser.add_argument('-d', '--datacenterid', dest='dc_id',
required=True, default=None,
help='datacenter ID of the server')
parser.add_argument('-o', '--outfile', dest='outfile',
default='dc-def_'+datetime.now().strftime('%Y-%m-%d_%H%M%S'),
help='the output file name')
parser.add_argument('-S', '--Stopalways', dest='stopalways', action='store_true',
help='power off even when VM is running')
parser.add_argument('-v', '--verbose', dest="verbose", action="count",
default=0, help="set verbosity level [default: %(default)s]")
parser.add_argument('-V', '--version', action='version',
version=program_version_message)
# Process arguments
args = parser.parse_args()
global verbose
verbose = args.verbose
if verbose > 0:
print("Verbose mode on")
print("start {} with args {}".format(program_name, str(args)))
outfile = args.outfile
if outfile.endswith(".json"):
outfile = os.path.splitext(outfile)
print("Using output file base name '{}'".format(outfile))
(user, password) = getLogin(args.loginfile, args.user, args.password)
if user is None or password is None:
raise ValueError("user or password resolved to None")
pbclient = ProfitBricksService(user, password)
dc_id = args.dc_id
# first get all server's VM and OS state to see if we can start
srv_info = getServerInfo(pbclient, dc_id)
srvon = 0
for server in srv_info:
if server['vmstate'] != 'SHUTOFF':
print("VM {} is in state {}, but should be SHUTOFF"
.format(server['name'], server['vmstate']))
srvon += 1
# end for(srv_info)
if srvon > 0 and not args.stopalways:
print("shutdown running OS before trying again")
return 1
# now power off all VMs before starting the snapshots
for server in srv_info:
controlServerState(pbclient, dc_id, server['id'], action='POWEROFF')
# now let's go
dcdef = pbclient.get_datacenter(dc_id, 5)
print("starting dump of datacenter {}".format(dcdef['properties']['name']))
dcdef_file = outfile+'_source.json'
print("write source dc to {}".format(dcdef_file))
write_dc_definition(pbclient, dcdef, dcdef_file)
print("get existing Snapshots")
# first get existing snapshots
known_snapshots = dict()
snapshots = pbclient.list_snapshots()
for snap in snapshots['items']:
print("SNAP : {}".format(json.dumps(snap)))
known_snapshots[snap['properties']['name']] = snap['id']
print("create Snapshots, this may take a while ..")
# we do NOT consider dangling volumes, only server-attached ones
vol_snapshots = dict() # map volume id==snapshot name snapshot id
for server in dcdef['entities']['servers']['items']:
print("- server {}".format(server['properties']['name']))
if 'volumes' not in server['entities']:
print(" server {} has no volumes"
.format(server['properties']['name']))
continue
# The volumes are attached by order of creation
# Thus we must sort them to keep the order in the clone
print("setting volume order by deviceNumber")
volumes = server['entities']['volumes']['items']
new_order = sorted(volumes, key=lambda vol: vol['properties']['deviceNumber'])
server['entities']['volumes']['items'] = new_order
for volume in server['entities']['volumes']['items']:
vol_id = volume['id'] # this will be the name too
if vol_id in known_snapshots:
print("use existing snapshot {} of volume {}"
.format(vol_id, volume['properties']['name']))
vol_snapshots[vol_id] = known_snapshots[vol_id]
else:
print("taking snapshot {} of volume {}"
.format(vol_id, volume['properties']['name']))
response = pbclient.create_snapshot(dc_id, vol_id, vol_id,
"auto-created by pb_snapshotDatacenter")
# response has no request id, need to check metadata state (BUSY, AVAILABLE..)
vol_snapshots[vol_id] = response['id']
print("snapshot in progress: {}".format(str(response)))
# end for(volume)
# end for(server)
print("Waiting for snapshots to complete")
snapdone = dict()
while len(snapdone) != len(vol_snapshots):
sleep(10)
for snap_id in vol_snapshots.values():
print("looking for {}".format(snap_id))
if snap_id in snapdone:
continue
snapshot = pbclient.get_snapshot(snap_id)
print("snapshot {} is in state {}"
.format(snap_id, snapshot['metadata']['state']))
if snapshot['metadata']['state'] == 'AVAILABLE':
snapdone[snap_id] = snapshot['metadata']['state']
# end for(vol_snapshots)
# end while(snapdone)
# now replace the volumes image IDs
print("setting snapshot id to volumes")
for server in dcdef['entities']['servers']['items']:
print("- server {}".format(server['properties']['name']))
if 'volumes' not in server['entities']:
print(" server {} has no volumes"
.format(server['properties']['name']))
continue
for volume in server['entities']['volumes']['items']:
vol_id = volume['id'] # this will be the name too
volume['properties']['image'] = vol_snapshots[vol_id]
# end for(volume)
# end for(server)
# As it came out, the LAN id is rearranged by order of creation
# Thus we must sort the LANs to keep the order in the clone
print("setting LAN order by id")
lans = dcdef['entities']['lans']['items']
new_order = sorted(lans, key=lambda lan: lan['id'])
dcdef['entities']['lans']['items'] = new_order
# now sort unordered NICs by MAC and save the dcdef
# reason is, that NICs seem to be ordered by MAC, but API response
# doesn't guarantee the order, which we need for re-creation
print("setting NIC order by MAC")
for server in dcdef['entities']['servers']['items']:
print("- server {}".format(server['properties']['name']))
if 'nics' not in server['entities']:
print(" server {} has no nics"
.format(server['properties']['name']))
continue
nics = server['entities']['nics']['items']
# print("NICs before {}".format(json.dumps(nics)))
new_order = sorted(nics, key=lambda nic: nic['properties']['mac'])
# print("NICs after {}".format(json.dumps(new_order)))
server['entities']['nics']['items'] = new_order
# end for(server)
dcdef_file = outfile+'.json'
print("write snapshot dc to {}".format(dcdef_file))
write_dc_definition(pbclient, dcdef, dcdef_file)
return 0
except KeyboardInterrupt:
# handle keyboard interrupt
return 0
except Exception:
traceback.print_exc()
sys.stderr.write("\n" + program_name + ": for help use --help\n")
return 2
