def __init__(self, host, port, proxy):
self.HOST = host
self.PORT = port
self.PROXY = proxy
self._conn = httplib.HTTPSConnection(host, port, cert_file=proxy, key_file=proxy)
log.debug('Created Httpsconnection with... HOST:%s PORT:%s PROXY:%s' %
(self.HOST, self.PORT, self.PROXY))
def label_image(self, path, label='/'):
cmd = ['tune2fs', '-L', label, path]
log.debug("Labeling image: '%s'" % cmd)
if subprocess.Popen(cmd, shell=False, env=config.get_restricted_env()).wait():
log.error("Unable to label image")
raise ImageUtilError("Unable to label image")
log.debug("%s labeled as %s" % (path, label))
def create_image(self, imagepath, size=None):
base_dir = os.path.dirname(imagepath)
if not os.path.exists(base_dir):
os.makedirs(base_dir)
log.debug("Created image base dir: '%s'" % imagepath)
root_stats = self.statvfs()
snapshot_stats = self.statvfs(base_dir)
# See if there is enough free space to create a snapshot (>50% free)
if root_stats['used'] > snapshot_stats['free']:
log.error("Not enought free space. (used:%s free:%s)" %
(root_stats['used'], snapshot_stats['free']))
raise ImageUtilError("ERROR. not enough free space")
# If specified, see if the requested size is enough
if size:
if size < root_stats['used']:
log.error("Specified Image size less then required")
log.error("Required:%s Requested:%s" % (root_stats['used'], size))
raise ImageUtilError("Specified partition size is less then needed.")
else:
size = root_stats['size']
self.dd_sparse(imagepath, size)
if self.partition:
self.create_bootable_partition(imagepath)
self.create_device_map(imagepath)
label = self.get_fs_label('/')
if label == None:
raise ImageUtilError("Your VM is partitioned but the partition where / is mounted is not labeled. Please see the repoman manpages for more information about the requirements for partitioned images.")
self.mkfs(self.device_map, label=label, fs_type = self.detect_fs_type('/'))
self.delete_device_map(imagepath)
else:
self.mkfs(imagepath, fs_type = self.detect_fs_type('/'))
def sync_fs(self, verbose):
#TODO: add progress bar into rsync somehow
log.info("Starting Sync Process")
exclude_list = ""
if self.system_excludes != None and len(self.system_excludes) > 0:
for exclude_item in self.system_excludes:
exclude_list += '--exclude "%s" ' % (exclude_item)
if self.user_excludes != None and len(self.user_excludes) > 0:
for exclude_item in self.user_excludes:
exclude_list += '--exclude "%s" ' % (exclude_item)
# Let's not forget to add the --delete-exclude flag to rsync else
# previously synced files which now match an exclude rule will not get
# deleted and will stay in the synced image.
if len(exclude_list) > 0:
exclude_list += " --delete-excluded"
flags = ''
if verbose:
flags += '--stats --progress '
cmd = "rsync -a --sparse %s --delete %s / %s" % (flags, exclude_list, self.mountpoint)
log.debug("%s" % cmd)
p = subprocess.Popen(cmd, shell=True, env=config.get_restricted_env()).wait()
if p:
log.error("Rsync encountered an issue. return code: '%s'" % p)
raise ImageUtilError("Rsync failed. Aborting.")
log.info("Sync Complete")
def validate_args(self, args):
# Default implementation is to do no validation on the arguments.
# Override this method in the child class where needed and it will
# automatically get called.
#
# Implementations of this method should throw a RepomanInvalidArgument
# exception when an argument fails validation.
log.debug('No argument validation implemented for the %s subcommand.' % (self.command))
def _json(self, resp):
body = resp.read()
log.debug("Message body from server: '%s'" % body)
try:
return json.loads(body)
except:
message = "Unable to parse response."
raise FormattingError(message, body)
def delegator(self, args):
log.info('repoman subcommand called: %s' % (self.command))
log.debug('args: %s' % (args))
# Validate CLI arguments first
log.debug('Validating arguments...')
self.validate_args(args)
# Now we can delegate to the child class
self.__call__(args)
def dd_sparse(self, path, size_bytes):
cmd = ['dd', 'if=/dev/zero', 'of=%s' % path, 'count=0', 'bs=1', 'seek=%s' % size_bytes]
log.debug("Creating sparse file: '%s'" % cmd)
null_f = open('/dev/null', 'w')
if subprocess.Popen(cmd, shell=False, stdout=null_f, stderr=null_f, env=config.get_restricted_env()).wait():
log.error("Unable to create sparse file")
raise ImageUtilError("Error creating sparse file")
null_f.close()
def mkfs(self, path, fs_type='ext3', label='/'):
if fs_type == None:
fs_type = 'ext3' # Default to ext3 if autodetection failed.
cmd = ['mkfs', '-t', fs_type, '-I', '128', '-F', '-L', label, path]
log.debug("Creating file system: '%s'" % cmd)
null_f = open('/dev/null', 'w')
if subprocess.Popen(cmd, shell=False, stdout=null_f, stderr=null_f, env=config.get_restricted_env()).wait():
log.error("Unable to create filesystem")
raise ImageUtilError("Error creating filesystem.")
null_f.close()
def check_mounted(self):
for line in open("/etc/mtab"):
fields = line.split(' ')
if self.partition and (fields[0] == self.device_map):
log.debug("Found image mounted in mtab: '%s'" % line)
return True
elif self.imagepath in line or (fields[1] == self.mountpoint):
log.debug("Found image mounted in mtab: '%s'" % line)
return True
return False
def setup_grub_conf(self, hypervisor):
try:
log.debug('Setting grub.conf for hypervisor %s' % (hypervisor))
self.mount_image()
grub_conf = os.path.join(self.mountpoint, 'boot', 'grub', 'grub.conf')
hypervisor_grub_conf = os.path.join(self.mountpoint, 'boot', 'grub', 'grub.conf-%s' % hypervisor)
if os.path.exists(hypervisor_grub_conf):
shutil.copy2(hypervisor_grub_conf, grub_conf)
log.debug('%s copied over %s' % (hypervisor_grub_conf, grub_conf))
else:
raise ImageUtilError('Could not locate %s' % (hypervisor_grub_conf))
finally:
self.umount_image()
def create_bootable_partition(self, path):
cmd = ['sfdisk', path]
log.debug("Creating bootable partition on %s" % (path))
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
if not p:
log.error("Error calling: %s" % (cmd))
raise ImageUtilError("Error creating bootable partition.")
p.stdin.write(',,L,*\n')
p.stdin.close()
p.wait()
if p.returncode != 0:
log.error("Command to create bootable partition returned error: %d" % (p.returncode))
raise ImageUtilError("Error creating bootable partition.")
log.debug("Bootable partition created on %s" % (path))
def get_partition_for_fs(self, fs):
"""
Returns the disk for a given fs.
"""
cmd = ['df', fs]
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
if not p:
log.error("Error calling: %s" % (cmd))
raise ImageUtilError("Error getting partition for filesystem %s" % (fs))
stdout = p.communicate()[0]
log.debug("[%s] output:\n%s" % (cmd, stdout))
feilds = stdout.split('\n')[1].split()
log.debug("Filesystem %s is on partition %s" % (fs, feilds[0]))
return feilds[0]
def recreate(self, origin, dest_root):
if not os.path.exists(origin):
return
stats = self.stat(origin)
dest = os.path.join(dest_root, origin.lstrip('/'))
if os.path.exists(dest):
self.destroy_files(dest)
if os.path.isdir(origin):
os.mkdir(dest)
elif os.path.isfile(origin):
open(dest).close()
os.chmod(dest, stats['mode'])
os.chown(dest, stats['uid'], stats['gid'])
log.debug("Recreated '%s' at '%s' (uid:%s,gid:%s,mode:%s)"
% (origin, dest, stats['uid'], stats['gid'], stats['mode']))
def install_mbr(self, path):
cmd = ['grub']
log.debug("Creating MBR on %s" % (path))
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
if not p:
log.error("Error calling: %s" % (cmd))
raise ImageUtilError("Error creating MBR on %s." % (path))
p.stdin.write('device (hd0) %s\n' % (path))
p.stdin.write('root (hd0,0)\n')
p.stdin.write('setup (hd0)\n')
p.stdin.write('quit\n')
p.stdin.close()
p.wait()
if p.returncode == 0:
log.debug("MBR created on %s" % (path))
else:
log.error("Error creating MBR on %s\nReturn code: %d" % (path, p.returncode))
raise ImageUtilError("Error creating MBR.")
def __call__(self, args):
# Check if sudo...
if os.getuid() != 0:
raise SubcommandFailure(self, "Error. This command requires root privileges, try again with sudo.")
kwargs={}
name = args.image
# Check for proper Gzip extension (if needed)
if args.gzip:
if name and name.endswith('.gz'):
pass
else:
log.info("Enforcing '.gz' extension.")
name += '.gz'
print ("WARNING: gzip option found, but your image name does not"
" end in '.gz'. Modifying image name to enforce this.")
print "New image name: '%s'" % (name)
kwargs['name'] = name
# Check to see if the image name has the '.gz' suffix and
# compression not enabled. Having uncompressed images named with
# a '.gz' suffix will break things in Nimbus.
if (not args.gzip) and name.endswith('.gz'):
print ("WARNING: The image name you gave ends with '.gz' but you did not enable image compression with the --gzip command line argument. Having uncompressed images with a '.gz' suffix to the image name can cause problems in some cloud frameworks.")
if not yes_or_no('Do you want to continue? [yes]/[n]o: '):
print "Aborting. Please select a new image name or enable compression via the --gzip command line argument."
return
exists = False
try:
image = self.get_repoman_client(args).describe_image(name, args.owner)
if image:
log.info("Found existing image")
exists = True
except RepomanError,e:
if e.status == 404:
log.debug("Did not find an existing image in repository with same name.")
pass
else:
log.error("Unexpected response occurred when testing if image exists.")
log.error("%s" % e)
raise SubcommandFailure(self, "Unexpected response from server occurred when testing if image exists.", e)
def _snapshot_system(self, start_fresh=False, verbose=False, clean=False):
exists = self.image_exists()
if clean:
log.info("Cleaning existing snapshot first.")
start_fresh=True
elif not exists:
log.info("No existing image found, creating a new one")
start_fresh=True
elif exists:
log.info("Existing image found, attempting to use.")
image_stats = self.stat(self.imagepath)
log.debug("Image stats:\n%s" % (image_stats))
if self.imagesize:
# requested size does not match current size
if self.imagesize != image_stats['size']:
log.warning("Requested size (%d) does not match current file (%d). Starting from scratch." % (self.imagesize, image_stats['size']))
start_fresh = True
else:
# image size does not match partition size
if image_stats['size'] != self.statvfs()['size']:
log.warning("Root partition size does not match image size. Starting from scratch")
start_fresh = True
if start_fresh:
# makesure image is unmounted, then destroy and recreate image.
log.info("Unmounting Image")
self.umount_image()
log.info("Destroying old files")
self.destroy_files(self.imagepath, self.mountpoint)
log.info("Creating new image")
self.create_image(self.imagepath, self.imagesize)
try:
log.info("Syncing file system")
self.mount_image()
self.sync_fs(verbose)
self.umount_image()
except ImageUtilError, e:
# Cleanup after failed sync
self.umount_image()
self.destroy_files(self.imagepath, self.mountpoint)
raise e
def uploaded_image_exist(self, image, owner, hypervisor='xen'):
"""
This method will test if an image has an uploaded file for a given hypervisor.
"""
log.info("Checking to see if image %s, owner %s, has an uploaded file for hypervisor %s" % (image, owner, hypervisor))
if owner:
resp = self._get('/api/images/%s/%s' % (owner, image))
else:
resp = self._get('/api/images/%s' % (image))
if resp.status != 200:
log.info("Image slot does not yet exist.")
raise RepomanError('Image does not yet exist.', resp)
if owner:
url = 'https://' + config.host + '/api/images/raw/%s/%s/%s' % (owner, hypervisor, image)
else:
url = 'https://' + config.host + '/api/images/raw/%s/%s' % (hypervisor, image)
try:
cmd = ['curl',
'--cert', config.proxy,
'--insecure',
'--head',
url]
log.debug(" ".join(cmd))
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
if not p:
log.error("Error calling: %s" % (cmd))
raise RepomanError("Error checking if image %s, owner %s, has an uploaded file for hypervisor %s" % (image, owner, hypervisor))
log.info("Command complete")
stdout = p.communicate()[0]
log.debug(stdout)
m = re.search('^HTTP/.+200 OK', stdout, flags=re.M)
if m:
log.info("Uploaded file exist for image %s, owner %s, hypervisor %s." % (image, owner, hypervisor))
return True
else:
log.info("Uploaded file does not exist for image %s, owner %s, hypervisor %s." % (image, owner, hypervisor))
return False
except Exception, e:
log.error("%s" % e)
raise RepomanError(str(e))
def get_fs_label(self, fs = '/'):
"""
Returns the given filesystem's label, or None if the filesystem has no label.
"""
log.debug("Detecting label for %s ..." % (fs))
# First detect the filesystem's partition.
partition = self.get_partition_for_fs(fs)
# Now use tune2fs to extract that partition's label
cmd = ['tune2fs', '-l', partition]
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
if not p:
log.error("Error calling: %s" % (cmd))
raise ImageUtilError("Error getting label for partition %s" % (partition))
stdout = p.communicate()[0]
log.debug("[%s] output:\n%s" % (cmd, stdout))
label = None
for l in stdout.split('\n'):
if l.startswith('Filesystem volume name'):
label = l.split(':')[1].strip()
if label == '<none>':
log.debug("Filesystem %s has no label." % (partition))
label = None
break
return label
def snapshot_system(self, start_fresh=False, verbose=False, clean=False):
if verbose:
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
formatter = logging.Formatter("%(levelname)s - %(message)s")
ch.setFormatter(formatter)
log.addHandler(ch)
log.debug("Obtaining lock")
if not self.obtain_lock():
log.error("Unable to obtain lock")
raise ImageUtilError("Unable to obtain lock")
snapshot_success = False
try:
self._snapshot_system(start_fresh, verbose, clean)
snapshot_success = True
finally:
log.debug("Releasing lock")
self.destroy_lock()
log.info("Unmounting Image")
self.umount_image()
if self.partition and snapshot_success:
self.install_mbr(self.imagepath)
def _request(self, method, url, kwargs={}, headers=HEADERS):
log.debug("%s %s" % (method, url))
log.debug("kwargs: %s" % kwargs)
log.debug("headers: %s" % headers)
try:
if method == 'GET':
self._conn.request(method, url)
elif method == 'DELETE':
self._conn.request(method, url)
elif method == 'POST':
params = urllib.urlencode(kwargs)
self._conn.request(method, url, params, headers)
resp = self._conn.getresponse()
log.debug("Server response code: %s" % resp.status)
return self._check_response(resp)
except RepomanError, e:
raise(e)
def mount_image(self):
if self.check_mounted():
log.debug('Image Already Mounted')
return
if not os.path.exists(self.mountpoint):
log.debug("Creating mount point")
os.makedirs(self.mountpoint)
cmd = None
if self.partition:
if not self.device_map:
self.create_device_map(self.imagepath)
cmd = ['mount', self.device_map, self.mountpoint]
else:
cmd = ['mount', '-o', 'loop', self.imagepath, self.mountpoint]
log.debug("running [%s]" % (cmd))
if subprocess.Popen(cmd, shell=False, env=config.get_restricted_env()).wait():
raise ImageUtilError("Unable to Mount image")
log.debug("Image mounted: '%s'" % cmd)
def umount_image(self):
if not self.check_mounted():
log.debug('Image already unmounted')
return
cmd = ['umount', self.mountpoint]
if subprocess.Popen(cmd, shell=False, env=config.get_restricted_env()).wait():
raise ImageUtilError("Unable to unmount image")
log.debug("Image unmounted: '%s'" % cmd)
if self.partition and self.device_map != None:
log.debug("Deleting %s device map for image %s" % (self.device_map, self.imagepath))
self.delete_device_map(self.imagepath)
def is_disk_partitioned(self):
"""
Detects if the disk is partitioned or not.
Returns True if the disk is partitioned, False otherwise.
"""
cmd = ['df', '/']
log.debug("Checking if disk is partitioned...")
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
if not p:
log.error("Error calling: %s" % (cmd))
raise ImageUtilError("Error checking if disk is partitioned.")
stdout = p.communicate()[0]
log.debug("[%s] output:\n%s" % (cmd, stdout))
feilds = stdout.split('\n')[1].split()
if feilds[0][-1].isdigit():
log.debug('Disk is partitioned.')
return True
else:
log.debug('Disk is not partitioned.')
return False
def delete_device_map(self, path):
cmd = ['kpartx', '-d', path]
log.debug("Deleting device map for %s" % (path))
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
if not p:
log.error("Error calling: %s" % (cmd))
raise ImageUtilError("Error deleting device map.")
stdout = p.communicate()[0]
log.debug("[%s] output:\n%s" % (cmd, stdout))
if p.returncode != 0:
log.error("Error deleting device map for %s" % (path))
raise ImageUtilError("Error deleting device map.")
log.debug("Device map deleted for %s" % (path))
self.device_map = None
def create_device_map(self, path):
if self.device_map != None:
log.error("Attempt to create device map over existing one. Aborting.")
raise ImageUtilError("Error creating device map.")
cmd = ['kpartx', '-av' , path]
log.debug("Creating device map for %s" % (path))
log.debug(cmd)
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
if not p:
log.error("Error calling: %s" % (cmd))
raise ImageUtilError("Error creating device map.")
(stdout, stderr) = p.communicate()
log.debug("[%s] kpartx output stdout:\n%s\n\nkpartx output stderr:\n%s" % (cmd, stdout, stderr))
if p.returncode != 0:
log.error("Device map creation command returned error: %d" % (p.returncode))
raise ImageUtilError("Error creating device map.")
# Search the output to extract the location of the new device map
m = re.search('^add map (\w+) .+$', stdout, flags=re.M)
if not m:
log.error("Error extracting location of new device map from:\n%s" % (stdout))
raise ImageUtilError("Error extracting location of new device map.")
log.debug("Device map created for %s" % (path))
self.device_map = '/dev/mapper/%s' % (m.group(1))
return self.device_map
def __call__(self, args):
#
# TODO: We need to implement some smarter server-side image listing
# commands and then cleanup this method to use these new functions.
# (Andre)
#
images = []
images_metadata = []
full_output = False
try:
if args.image:
image_name = args.image
full_output = True
if args.owner:
image_name = "%s/%s" % (args.owner, args.image)
log.debug('Fetching info of single image "%s"' % (image_name))
images_metadata.append(self.get_repoman_client(args).describe_image(image_name))
else:
if args.all:
# List images that the user has access to.
# (either via ownership, or shared by user or group membership.)
# For a repoman admin, this will be all images on the server.
#func = repo.list_all_images
log.debug('Listing all images...')
kwargs = {}
images = self.get_repoman_client(args).list_current_user_images(**kwargs)
images += self.get_repoman_client(args).list_images_shared_with_user(**kwargs)
elif args.group:
# List images accessible by you and by members of the named group.
# First check if the user is a member of the group. If not, then
# return an empty list.
log.debug('Listing all images for group "%s"' % (args.group))
kwargs = {'group':args.group}
groups = self.get_repoman_client(args).whoami()['groups']
for group in groups:
if group.split('/')[-1] == args.group:
images = self.get_repoman_client(args).list_images_shared_with_group(**kwargs)
elif args.user:
log.debug('Listing all current user\'s images shared with user %s' % (args.user))
current_user = self.get_repoman_client(args).whoami()['user_name']
# List all current user's images shared with the given user, AND all of the
# given user's images shared with the current user.
kwargs = {'user':args.user}
all_images_shared_with_given_user = self.get_repoman_client(args).list_images_shared_with_user(**kwargs)
for image in all_images_shared_with_given_user:
if image.split('/')[-2] == current_user:
images.append(image)
kwargs = {'user':current_user}
all_images_shared_with_me = self.get_repoman_client(args).list_images_shared_with_user(**kwargs)
for image in all_images_shared_with_me:
if image.split('/')[-2] == args.user:
images.append(image)
else:
# List only images owned by current user.
log.debug('Listing images owned by current user.')
kwargs = {}
images = self.get_repoman_client(args).list_current_user_images(**kwargs)
# Get the metadata of each image.
# TODO: This is a non-efficient hack that will be cleaned-up later. (Andre)
log.debug('Images:\n%s\n' % (images))
for image in images:
try:
name = image.rsplit('/', 2)
log.debug('Fetching image metadata for "%s" [%s]' % (image, name))
images_metadata.append(self.get_repoman_client(args).describe_image("%s/%s" % (name[-2], name[-1])))
except RepomanError, e:
print "Warning: Error in retreiving information about image '%s'. Skipping..." % (image)
except Exception, e:
raise RepomanError(self, "Error in retreiving information about image '%s'." % (image), e)
def destroy_files(self, *args):
cmd = ['rm', '-rf'] + list(args)
subprocess.call(cmd, shell=False, env=config.get_restricted_env())
log.debug("Destroyed files: '%s'" % cmd)
