def __init__(self, host, port, proxy): self.HOST = host self.PORT = port self.PROXY = proxy self._conn = httplib.HTTPSConnection(host, port, cert_file=proxy, key_file=proxy) log.debug('Created Httpsconnection with... HOST:%s PORT:%s PROXY:%s' % (self.HOST, self.PORT, self.PROXY))
def label_image(self, path, label='/'): cmd = ['tune2fs', '-L', label, path] log.debug("Labeling image: '%s'" % cmd) if subprocess.Popen(cmd, shell=False, env=config.get_restricted_env()).wait(): log.error("Unable to label image") raise ImageUtilError("Unable to label image") log.debug("%s labeled as %s" % (path, label))
def create_image(self, imagepath, size=None): base_dir = os.path.dirname(imagepath) if not os.path.exists(base_dir): os.makedirs(base_dir) log.debug("Created image base dir: '%s'" % imagepath) root_stats = self.statvfs() snapshot_stats = self.statvfs(base_dir) # See if there is enough free space to create a snapshot (>50% free) if root_stats['used'] > snapshot_stats['free']: log.error("Not enought free space. (used:%s free:%s)" % (root_stats['used'], snapshot_stats['free'])) raise ImageUtilError("ERROR. not enough free space") # If specified, see if the requested size is enough if size: if size < root_stats['used']: log.error("Specified Image size less then required") log.error("Required:%s Requested:%s" % (root_stats['used'], size)) raise ImageUtilError("Specified partition size is less then needed.") else: size = root_stats['size'] self.dd_sparse(imagepath, size) if self.partition: self.create_bootable_partition(imagepath) self.create_device_map(imagepath) label = self.get_fs_label('/') if label == None: raise ImageUtilError("Your VM is partitioned but the partition where / is mounted is not labeled. Please see the repoman manpages for more information about the requirements for partitioned images.") self.mkfs(self.device_map, label=label, fs_type = self.detect_fs_type('/')) self.delete_device_map(imagepath) else: self.mkfs(imagepath, fs_type = self.detect_fs_type('/'))
def sync_fs(self, verbose): #TODO: add progress bar into rsync somehow log.info("Starting Sync Process") exclude_list = "" if self.system_excludes != None and len(self.system_excludes) > 0: for exclude_item in self.system_excludes: exclude_list += '--exclude "%s" ' % (exclude_item) if self.user_excludes != None and len(self.user_excludes) > 0: for exclude_item in self.user_excludes: exclude_list += '--exclude "%s" ' % (exclude_item) # Let's not forget to add the --delete-exclude flag to rsync else # previously synced files which now match an exclude rule will not get # deleted and will stay in the synced image. if len(exclude_list) > 0: exclude_list += " --delete-excluded" flags = '' if verbose: flags += '--stats --progress ' cmd = "rsync -a --sparse %s --delete %s / %s" % (flags, exclude_list, self.mountpoint) log.debug("%s" % cmd) p = subprocess.Popen(cmd, shell=True, env=config.get_restricted_env()).wait() if p: log.error("Rsync encountered an issue. return code: '%s'" % p) raise ImageUtilError("Rsync failed. Aborting.") log.info("Sync Complete")
def validate_args(self, args): # Default implementation is to do no validation on the arguments. # Override this method in the child class where needed and it will # automatically get called. # # Implementations of this method should throw a RepomanInvalidArgument # exception when an argument fails validation. log.debug('No argument validation implemented for the %s subcommand.' % (self.command))
def _json(self, resp): body = resp.read() log.debug("Message body from server: '%s'" % body) try: return json.loads(body) except: message = "Unable to parse response." raise FormattingError(message, body)
def delegator(self, args): log.info('repoman subcommand called: %s' % (self.command)) log.debug('args: %s' % (args)) # Validate CLI arguments first log.debug('Validating arguments...') self.validate_args(args) # Now we can delegate to the child class self.__call__(args)
def dd_sparse(self, path, size_bytes): cmd = ['dd', 'if=/dev/zero', 'of=%s' % path, 'count=0', 'bs=1', 'seek=%s' % size_bytes] log.debug("Creating sparse file: '%s'" % cmd) null_f = open('/dev/null', 'w') if subprocess.Popen(cmd, shell=False, stdout=null_f, stderr=null_f, env=config.get_restricted_env()).wait(): log.error("Unable to create sparse file") raise ImageUtilError("Error creating sparse file") null_f.close()
def mkfs(self, path, fs_type='ext3', label='/'): if fs_type == None: fs_type = 'ext3' # Default to ext3 if autodetection failed. cmd = ['mkfs', '-t', fs_type, '-I', '128', '-F', '-L', label, path] log.debug("Creating file system: '%s'" % cmd) null_f = open('/dev/null', 'w') if subprocess.Popen(cmd, shell=False, stdout=null_f, stderr=null_f, env=config.get_restricted_env()).wait(): log.error("Unable to create filesystem") raise ImageUtilError("Error creating filesystem.") null_f.close()
def check_mounted(self): for line in open("/etc/mtab"): fields = line.split(' ') if self.partition and (fields[0] == self.device_map): log.debug("Found image mounted in mtab: '%s'" % line) return True elif self.imagepath in line or (fields[1] == self.mountpoint): log.debug("Found image mounted in mtab: '%s'" % line) return True return False
def setup_grub_conf(self, hypervisor): try: log.debug('Setting grub.conf for hypervisor %s' % (hypervisor)) self.mount_image() grub_conf = os.path.join(self.mountpoint, 'boot', 'grub', 'grub.conf') hypervisor_grub_conf = os.path.join(self.mountpoint, 'boot', 'grub', 'grub.conf-%s' % hypervisor) if os.path.exists(hypervisor_grub_conf): shutil.copy2(hypervisor_grub_conf, grub_conf) log.debug('%s copied over %s' % (hypervisor_grub_conf, grub_conf)) else: raise ImageUtilError('Could not locate %s' % (hypervisor_grub_conf)) finally: self.umount_image()
def create_bootable_partition(self, path): cmd = ['sfdisk', path] log.debug("Creating bootable partition on %s" % (path)) p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env()) if not p: log.error("Error calling: %s" % (cmd)) raise ImageUtilError("Error creating bootable partition.") p.stdin.write(',,L,*\n') p.stdin.close() p.wait() if p.returncode != 0: log.error("Command to create bootable partition returned error: %d" % (p.returncode)) raise ImageUtilError("Error creating bootable partition.") log.debug("Bootable partition created on %s" % (path))
def get_partition_for_fs(self, fs): """ Returns the disk for a given fs. """ cmd = ['df', fs] p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env()) if not p: log.error("Error calling: %s" % (cmd)) raise ImageUtilError("Error getting partition for filesystem %s" % (fs)) stdout = p.communicate()[0] log.debug("[%s] output:\n%s" % (cmd, stdout)) feilds = stdout.split('\n')[1].split() log.debug("Filesystem %s is on partition %s" % (fs, feilds[0])) return feilds[0]
def recreate(self, origin, dest_root): if not os.path.exists(origin): return stats = self.stat(origin) dest = os.path.join(dest_root, origin.lstrip('/')) if os.path.exists(dest): self.destroy_files(dest) if os.path.isdir(origin): os.mkdir(dest) elif os.path.isfile(origin): open(dest).close() os.chmod(dest, stats['mode']) os.chown(dest, stats['uid'], stats['gid']) log.debug("Recreated '%s' at '%s' (uid:%s,gid:%s,mode:%s)" % (origin, dest, stats['uid'], stats['gid'], stats['mode']))
def install_mbr(self, path): cmd = ['grub'] log.debug("Creating MBR on %s" % (path)) p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env()) if not p: log.error("Error calling: %s" % (cmd)) raise ImageUtilError("Error creating MBR on %s." % (path)) p.stdin.write('device (hd0) %s\n' % (path)) p.stdin.write('root (hd0,0)\n') p.stdin.write('setup (hd0)\n') p.stdin.write('quit\n') p.stdin.close() p.wait() if p.returncode == 0: log.debug("MBR created on %s" % (path)) else: log.error("Error creating MBR on %s\nReturn code: %d" % (path, p.returncode)) raise ImageUtilError("Error creating MBR.")
def __call__(self, args): # Check if sudo... if os.getuid() != 0: raise SubcommandFailure(self, "Error. This command requires root privileges, try again with sudo.") kwargs={} name = args.image # Check for proper Gzip extension (if needed) if args.gzip: if name and name.endswith('.gz'): pass else: log.info("Enforcing '.gz' extension.") name += '.gz' print ("WARNING: gzip option found, but your image name does not" " end in '.gz'. Modifying image name to enforce this.") print "New image name: '%s'" % (name) kwargs['name'] = name # Check to see if the image name has the '.gz' suffix and # compression not enabled. Having uncompressed images named with # a '.gz' suffix will break things in Nimbus. if (not args.gzip) and name.endswith('.gz'): print ("WARNING: The image name you gave ends with '.gz' but you did not enable image compression with the --gzip command line argument. Having uncompressed images with a '.gz' suffix to the image name can cause problems in some cloud frameworks.") if not yes_or_no('Do you want to continue? [yes]/[n]o: '): print "Aborting. Please select a new image name or enable compression via the --gzip command line argument." return exists = False try: image = self.get_repoman_client(args).describe_image(name, args.owner) if image: log.info("Found existing image") exists = True except RepomanError,e: if e.status == 404: log.debug("Did not find an existing image in repository with same name.") pass else: log.error("Unexpected response occurred when testing if image exists.") log.error("%s" % e) raise SubcommandFailure(self, "Unexpected response from server occurred when testing if image exists.", e)
def _snapshot_system(self, start_fresh=False, verbose=False, clean=False): exists = self.image_exists() if clean: log.info("Cleaning existing snapshot first.") start_fresh=True elif not exists: log.info("No existing image found, creating a new one") start_fresh=True elif exists: log.info("Existing image found, attempting to use.") image_stats = self.stat(self.imagepath) log.debug("Image stats:\n%s" % (image_stats)) if self.imagesize: # requested size does not match current size if self.imagesize != image_stats['size']: log.warning("Requested size (%d) does not match current file (%d). Starting from scratch." % (self.imagesize, image_stats['size'])) start_fresh = True else: # image size does not match partition size if image_stats['size'] != self.statvfs()['size']: log.warning("Root partition size does not match image size. Starting from scratch") start_fresh = True if start_fresh: # makesure image is unmounted, then destroy and recreate image. log.info("Unmounting Image") self.umount_image() log.info("Destroying old files") self.destroy_files(self.imagepath, self.mountpoint) log.info("Creating new image") self.create_image(self.imagepath, self.imagesize) try: log.info("Syncing file system") self.mount_image() self.sync_fs(verbose) self.umount_image() except ImageUtilError, e: # Cleanup after failed sync self.umount_image() self.destroy_files(self.imagepath, self.mountpoint) raise e
def uploaded_image_exist(self, image, owner, hypervisor='xen'): """ This method will test if an image has an uploaded file for a given hypervisor. """ log.info("Checking to see if image %s, owner %s, has an uploaded file for hypervisor %s" % (image, owner, hypervisor)) if owner: resp = self._get('/api/images/%s/%s' % (owner, image)) else: resp = self._get('/api/images/%s' % (image)) if resp.status != 200: log.info("Image slot does not yet exist.") raise RepomanError('Image does not yet exist.', resp) if owner: url = 'https://' + config.host + '/api/images/raw/%s/%s/%s' % (owner, hypervisor, image) else: url = 'https://' + config.host + '/api/images/raw/%s/%s' % (hypervisor, image) try: cmd = ['curl', '--cert', config.proxy, '--insecure', '--head', url] log.debug(" ".join(cmd)) p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env()) if not p: log.error("Error calling: %s" % (cmd)) raise RepomanError("Error checking if image %s, owner %s, has an uploaded file for hypervisor %s" % (image, owner, hypervisor)) log.info("Command complete") stdout = p.communicate()[0] log.debug(stdout) m = re.search('^HTTP/.+200 OK', stdout, flags=re.M) if m: log.info("Uploaded file exist for image %s, owner %s, hypervisor %s." % (image, owner, hypervisor)) return True else: log.info("Uploaded file does not exist for image %s, owner %s, hypervisor %s." % (image, owner, hypervisor)) return False except Exception, e: log.error("%s" % e) raise RepomanError(str(e))
def get_fs_label(self, fs = '/'): """ Returns the given filesystem's label, or None if the filesystem has no label. """ log.debug("Detecting label for %s ..." % (fs)) # First detect the filesystem's partition. partition = self.get_partition_for_fs(fs) # Now use tune2fs to extract that partition's label cmd = ['tune2fs', '-l', partition] p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env()) if not p: log.error("Error calling: %s" % (cmd)) raise ImageUtilError("Error getting label for partition %s" % (partition)) stdout = p.communicate()[0] log.debug("[%s] output:\n%s" % (cmd, stdout)) label = None for l in stdout.split('\n'): if l.startswith('Filesystem volume name'): label = l.split(':')[1].strip() if label == '<none>': log.debug("Filesystem %s has no label." % (partition)) label = None break return label
def snapshot_system(self, start_fresh=False, verbose=False, clean=False): if verbose: ch = logging.StreamHandler() ch.setLevel(logging.INFO) formatter = logging.Formatter("%(levelname)s - %(message)s") ch.setFormatter(formatter) log.addHandler(ch) log.debug("Obtaining lock") if not self.obtain_lock(): log.error("Unable to obtain lock") raise ImageUtilError("Unable to obtain lock") snapshot_success = False try: self._snapshot_system(start_fresh, verbose, clean) snapshot_success = True finally: log.debug("Releasing lock") self.destroy_lock() log.info("Unmounting Image") self.umount_image() if self.partition and snapshot_success: self.install_mbr(self.imagepath)
def _request(self, method, url, kwargs={}, headers=HEADERS): log.debug("%s %s" % (method, url)) log.debug("kwargs: %s" % kwargs) log.debug("headers: %s" % headers) try: if method == 'GET': self._conn.request(method, url) elif method == 'DELETE': self._conn.request(method, url) elif method == 'POST': params = urllib.urlencode(kwargs) self._conn.request(method, url, params, headers) resp = self._conn.getresponse() log.debug("Server response code: %s" % resp.status) return self._check_response(resp) except RepomanError, e: raise(e)
def mount_image(self): if self.check_mounted(): log.debug('Image Already Mounted') return if not os.path.exists(self.mountpoint): log.debug("Creating mount point") os.makedirs(self.mountpoint) cmd = None if self.partition: if not self.device_map: self.create_device_map(self.imagepath) cmd = ['mount', self.device_map, self.mountpoint] else: cmd = ['mount', '-o', 'loop', self.imagepath, self.mountpoint] log.debug("running [%s]" % (cmd)) if subprocess.Popen(cmd, shell=False, env=config.get_restricted_env()).wait(): raise ImageUtilError("Unable to Mount image") log.debug("Image mounted: '%s'" % cmd)
def umount_image(self): if not self.check_mounted(): log.debug('Image already unmounted') return cmd = ['umount', self.mountpoint] if subprocess.Popen(cmd, shell=False, env=config.get_restricted_env()).wait(): raise ImageUtilError("Unable to unmount image") log.debug("Image unmounted: '%s'" % cmd) if self.partition and self.device_map != None: log.debug("Deleting %s device map for image %s" % (self.device_map, self.imagepath)) self.delete_device_map(self.imagepath)
def is_disk_partitioned(self): """ Detects if the disk is partitioned or not. Returns True if the disk is partitioned, False otherwise. """ cmd = ['df', '/'] log.debug("Checking if disk is partitioned...") p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env()) if not p: log.error("Error calling: %s" % (cmd)) raise ImageUtilError("Error checking if disk is partitioned.") stdout = p.communicate()[0] log.debug("[%s] output:\n%s" % (cmd, stdout)) feilds = stdout.split('\n')[1].split() if feilds[0][-1].isdigit(): log.debug('Disk is partitioned.') return True else: log.debug('Disk is not partitioned.') return False
def delete_device_map(self, path): cmd = ['kpartx', '-d', path] log.debug("Deleting device map for %s" % (path)) p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env()) if not p: log.error("Error calling: %s" % (cmd)) raise ImageUtilError("Error deleting device map.") stdout = p.communicate()[0] log.debug("[%s] output:\n%s" % (cmd, stdout)) if p.returncode != 0: log.error("Error deleting device map for %s" % (path)) raise ImageUtilError("Error deleting device map.") log.debug("Device map deleted for %s" % (path)) self.device_map = None
def create_device_map(self, path): if self.device_map != None: log.error("Attempt to create device map over existing one. Aborting.") raise ImageUtilError("Error creating device map.") cmd = ['kpartx', '-av' , path] log.debug("Creating device map for %s" % (path)) log.debug(cmd) p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env()) if not p: log.error("Error calling: %s" % (cmd)) raise ImageUtilError("Error creating device map.") (stdout, stderr) = p.communicate() log.debug("[%s] kpartx output stdout:\n%s\n\nkpartx output stderr:\n%s" % (cmd, stdout, stderr)) if p.returncode != 0: log.error("Device map creation command returned error: %d" % (p.returncode)) raise ImageUtilError("Error creating device map.") # Search the output to extract the location of the new device map m = re.search('^add map (\w+) .+$', stdout, flags=re.M) if not m: log.error("Error extracting location of new device map from:\n%s" % (stdout)) raise ImageUtilError("Error extracting location of new device map.") log.debug("Device map created for %s" % (path)) self.device_map = '/dev/mapper/%s' % (m.group(1)) return self.device_map
def __call__(self, args): # # TODO: We need to implement some smarter server-side image listing # commands and then cleanup this method to use these new functions. # (Andre) # images = [] images_metadata = [] full_output = False try: if args.image: image_name = args.image full_output = True if args.owner: image_name = "%s/%s" % (args.owner, args.image) log.debug('Fetching info of single image "%s"' % (image_name)) images_metadata.append(self.get_repoman_client(args).describe_image(image_name)) else: if args.all: # List images that the user has access to. # (either via ownership, or shared by user or group membership.) # For a repoman admin, this will be all images on the server. #func = repo.list_all_images log.debug('Listing all images...') kwargs = {} images = self.get_repoman_client(args).list_current_user_images(**kwargs) images += self.get_repoman_client(args).list_images_shared_with_user(**kwargs) elif args.group: # List images accessible by you and by members of the named group. # First check if the user is a member of the group. If not, then # return an empty list. log.debug('Listing all images for group "%s"' % (args.group)) kwargs = {'group':args.group} groups = self.get_repoman_client(args).whoami()['groups'] for group in groups: if group.split('/')[-1] == args.group: images = self.get_repoman_client(args).list_images_shared_with_group(**kwargs) elif args.user: log.debug('Listing all current user\'s images shared with user %s' % (args.user)) current_user = self.get_repoman_client(args).whoami()['user_name'] # List all current user's images shared with the given user, AND all of the # given user's images shared with the current user. kwargs = {'user':args.user} all_images_shared_with_given_user = self.get_repoman_client(args).list_images_shared_with_user(**kwargs) for image in all_images_shared_with_given_user: if image.split('/')[-2] == current_user: images.append(image) kwargs = {'user':current_user} all_images_shared_with_me = self.get_repoman_client(args).list_images_shared_with_user(**kwargs) for image in all_images_shared_with_me: if image.split('/')[-2] == args.user: images.append(image) else: # List only images owned by current user. log.debug('Listing images owned by current user.') kwargs = {} images = self.get_repoman_client(args).list_current_user_images(**kwargs) # Get the metadata of each image. # TODO: This is a non-efficient hack that will be cleaned-up later. (Andre) log.debug('Images:\n%s\n' % (images)) for image in images: try: name = image.rsplit('/', 2) log.debug('Fetching image metadata for "%s" [%s]' % (image, name)) images_metadata.append(self.get_repoman_client(args).describe_image("%s/%s" % (name[-2], name[-1]))) except RepomanError, e: print "Warning: Error in retreiving information about image '%s'. Skipping..." % (image) except Exception, e: raise RepomanError(self, "Error in retreiving information about image '%s'." % (image), e)
def destroy_files(self, *args): cmd = ['rm', '-rf'] + list(args) subprocess.call(cmd, shell=False, env=config.get_restricted_env()) log.debug("Destroyed files: '%s'" % cmd)