Пример #1
0
    def sync_fs(self, verbose):
        #TODO: add progress bar into rsync somehow
        log.info("Starting Sync Process")

        exclude_list = ""
        if self.system_excludes != None and len(self.system_excludes) > 0:
            for exclude_item in self.system_excludes:
                exclude_list += '--exclude "%s" ' % (exclude_item)
        if self.user_excludes != None and len(self.user_excludes) > 0:
            for exclude_item in self.user_excludes:
                exclude_list += '--exclude "%s" ' % (exclude_item)
        # Let's not forget to add the --delete-exclude flag to rsync else
        # previously synced files which now match an exclude rule will not get
        # deleted and will stay in the synced image.
        if len(exclude_list) > 0:
            exclude_list += " --delete-excluded"

        flags = ''
        if verbose:
            flags += '--stats --progress ' 
        cmd = "rsync -a --sparse %s --delete %s / %s" % (flags, exclude_list, self.mountpoint)
        log.debug("%s" % cmd)
        p = subprocess.Popen(cmd, shell=True, env=config.get_restricted_env()).wait()
        if p:
            log.error("Rsync encountered an issue. return code: '%s'" % p)
            raise ImageUtilError("Rsync failed.  Aborting.")
        log.info("Sync Complete")
Пример #2
0
    def get_fs_label(self, fs = '/'):
        """
        Returns the given filesystem's label, or None if the filesystem has no label.
        """
        log.debug("Detecting label for %s ..." % (fs))

        # First detect the filesystem's partition.
        partition = self.get_partition_for_fs(fs)

        # Now use tune2fs to extract that partition's label
        cmd = ['tune2fs', '-l', partition]
        p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
        if not p:
            log.error("Error calling: %s" % (cmd))
            raise ImageUtilError("Error getting label for partition %s" % (partition))

        stdout = p.communicate()[0]
        log.debug("[%s] output:\n%s" % (cmd, stdout))

        label = None
        for l in stdout.split('\n'):
            if l.startswith('Filesystem volume name'):
                label = l.split(':')[1].strip()
                if label == '<none>':
                    log.debug("Filesystem %s has no label." % (partition))
                    label = None
                break
        return label
Пример #3
0
 def label_image(self, path, label='/'):
     cmd = ['tune2fs', '-L', label, path]
     log.debug("Labeling image: '%s'" % cmd)
     if subprocess.Popen(cmd, shell=False, env=config.get_restricted_env()).wait():
         log.error("Unable to label image")
         raise ImageUtilError("Unable to label image")
     log.debug("%s labeled as %s" % (path, label))
Пример #4
0
 def dd_sparse(self, path, size_bytes):
     cmd = ['dd', 'if=/dev/zero', 'of=%s' % path, 'count=0', 'bs=1', 'seek=%s' % size_bytes]
     log.debug("Creating sparse file: '%s'" % cmd)
     null_f = open('/dev/null', 'w')
     if subprocess.Popen(cmd, shell=False, stdout=null_f, stderr=null_f, env=config.get_restricted_env()).wait():
         log.error("Unable to create sparse file")
         raise ImageUtilError("Error creating sparse file")
     null_f.close()
Пример #5
0
    def upload_image(self, image, owner, image_file, gzip=False, hypervisor='xen'):
        log.info("Checking to see if image slot exists on repository")
        if owner:
            resp = self._get('/api/images/%s/%s' % (owner, image))
        else:
            resp = self._get('/api/images/%s' % (image))

        if resp.status != 200:
            log.info("Image slot does not yet exist.")
            raise RepomanError('Image does not yet exist.  Create an image before uploading to it', resp)

        # Check if the source is a directory.  If it is, then raise an
        # exception.
        if os.path.isdir(image_file):
            raise RepomanError('Specified source is a directory: %s\nSource must be a file.' % (image_file))
           
        # Check if the source file exists.
        if not os.path.exists(image_file):
            raise RepomanError('Specified source not found: %s' % (image_file))
            
        if owner:
            url = 'https://' + config.host + '/api/images/raw/%s/%s/%s' % (owner, hypervisor, image)
        else:
            url = 'https://' + config.host + '/api/images/raw/%s/%s' % (hypervisor, image)

        try:
            if gzip:
                log.info("Performing gzip on image prior to upload")
                print "Gzipping image before upload"
                gzip_image = os.path.join(os.path.dirname(image_file), image)
                gzip = subprocess.Popen("gzip --stdout %s > %s" % (image_file, gzip_image),
                                        shell=True, env=config.get_restricted_env())
                gzip.wait()
                image_file = gzip_image
                log.info('Gzip complete')

            args = ['curl',
                    '--cert', config.proxy,
                    '--insecure',
                    '-T', image_file, url]
            cmd = " ".join(args)
            log.info("Running command: '%s'" % cmd)
            curl = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, env=config.get_restricted_env())
            for line in curl.stdout.readlines():
                print line
            log.info("Command complete")
            # Cleanup gzip file if needed.
            if gzip:
                try:
                    log.info("Cleaning up %s" % (image_file))
                    os.remove(image_file)
                except Exception, e:
                    pass
 
        except Exception, e:
            log.error("%s" % e)
            raise RepomanError(str(e))
Пример #6
0
 def mkfs(self, path, fs_type='ext3', label='/'):
     if fs_type == None:
         fs_type = 'ext3' # Default to ext3 if autodetection failed.
     cmd = ['mkfs', '-t', fs_type, '-I', '128', '-F', '-L', label, path]
     log.debug("Creating file system: '%s'" % cmd)
     null_f = open('/dev/null', 'w')
     if subprocess.Popen(cmd, shell=False, stdout=null_f, stderr=null_f, env=config.get_restricted_env()).wait():
         log.error("Unable to create filesystem")
         raise ImageUtilError("Error creating filesystem.")
     null_f.close()
Пример #7
0
 def delete_device_map(self, path):
     cmd = ['kpartx', '-d', path]
     log.debug("Deleting device map for %s" % (path))
     p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
     if not p:
         log.error("Error calling: %s" % (cmd))
         raise ImageUtilError("Error deleting device map.")
     stdout = p.communicate()[0]
     log.debug("[%s] output:\n%s" % (cmd, stdout))
     if p.returncode != 0:
         log.error("Error deleting device map for %s" % (path))
         raise ImageUtilError("Error deleting device map.")
     log.debug("Device map deleted for %s" % (path))
     self.device_map = None
Пример #8
0
 def create_bootable_partition(self, path):
     cmd = ['sfdisk', path]
     log.debug("Creating bootable partition on %s" % (path))
     p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
     if not p:
         log.error("Error calling: %s" % (cmd))
         raise ImageUtilError("Error creating bootable partition.")
     p.stdin.write(',,L,*\n')
     p.stdin.close()
     p.wait()
     if p.returncode != 0:
         log.error("Command to create bootable partition returned error: %d" % (p.returncode))
         raise ImageUtilError("Error creating bootable partition.")
     log.debug("Bootable partition created on %s" % (path))
Пример #9
0
    def get_partition_for_fs(self, fs):
        """
        Returns the disk for a given fs.
        """
        cmd = ['df', fs]
        p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
        if not p:
            log.error("Error calling: %s" % (cmd))
            raise ImageUtilError("Error getting partition for filesystem %s" % (fs))

        stdout = p.communicate()[0]
        log.debug("[%s] output:\n%s" % (cmd, stdout))
        feilds = stdout.split('\n')[1].split()
        log.debug("Filesystem %s is on partition %s" % (fs, feilds[0]))
        return feilds[0]
Пример #10
0
 def install_mbr(self, path):
     cmd = ['grub']
     log.debug("Creating MBR on %s" % (path))
     p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
     if not p:
         log.error("Error calling: %s" % (cmd))
         raise ImageUtilError("Error creating MBR on %s." % (path))
     p.stdin.write('device (hd0) %s\n' % (path))
     p.stdin.write('root (hd0,0)\n')
     p.stdin.write('setup (hd0)\n')
     p.stdin.write('quit\n')
     p.stdin.close()
     p.wait()
     if p.returncode == 0:
         log.debug("MBR created on %s" % (path))
     else:
         log.error("Error creating MBR on %s\nReturn code: %d" % (path, p.returncode))
         raise ImageUtilError("Error creating MBR.")
Пример #11
0
    def __call__(self, args):
        # Check if sudo...
        if os.getuid() != 0:
            raise SubcommandFailure(self, "Error.  This command requires root privileges, try again with sudo.")

        kwargs={}

        name = args.image
        # Check for proper Gzip extension (if needed)
        if args.gzip:
            if name and name.endswith('.gz'):
                pass
            else:
                log.info("Enforcing '.gz' extension.")
                name += '.gz'
                print ("WARNING: gzip option found, but your image name does not"
                       " end in '.gz'.  Modifying image name to enforce this.")
                print "New image name: '%s'" % (name)

        kwargs['name'] = name

        # Check to see if the image name has the '.gz' suffix and
        # compression not enabled.  Having uncompressed images named with
        # a '.gz' suffix will break things in Nimbus.
        if (not args.gzip) and name.endswith('.gz'):
            print ("WARNING: The image name you gave ends with '.gz' but you did not enable image compression with the --gzip command line argument.  Having uncompressed images with a '.gz' suffix to the image name can cause problems in some cloud frameworks.")
            if not yes_or_no('Do you want to continue? [yes]/[n]o: '):
                print "Aborting.  Please select a new image name or enable compression via the --gzip command line argument."
                return

        exists = False
        try:
            image = self.get_repoman_client(args).describe_image(name, args.owner)
            if image:
                log.info("Found existing image")
                exists = True
        except RepomanError,e:
            if e.status == 404:
                log.debug("Did not find an existing image in repository with same name.")
                pass
            else:
                log.error("Unexpected response occurred when testing if image exists.")
                log.error("%s" % e)
                raise SubcommandFailure(self, "Unexpected response from server occurred when testing if image exists.", e)
Пример #12
0
 def is_disk_partitioned(self):
     """
     Detects if the disk is partitioned or not.
     Returns True if the disk is partitioned, False otherwise.
     """
     cmd = ['df', '/']
     log.debug("Checking if disk is partitioned...")
     p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
     if not p:
         log.error("Error calling: %s" % (cmd))
         raise ImageUtilError("Error checking if disk is partitioned.")
     stdout = p.communicate()[0]
     log.debug("[%s] output:\n%s" % (cmd, stdout))
     feilds = stdout.split('\n')[1].split()
     if feilds[0][-1].isdigit():
         log.debug('Disk is partitioned.')
         return True
     else:
         log.debug('Disk is not partitioned.')
         return False
Пример #13
0
    def uploaded_image_exist(self, image, owner, hypervisor='xen'):
        """
        This method will test if an image has an uploaded file for a given hypervisor.
        """
        log.info("Checking to see if image %s, owner %s, has an uploaded file for hypervisor %s" % (image, owner, hypervisor))
        if owner:
            resp = self._get('/api/images/%s/%s' % (owner, image))
        else:
            resp = self._get('/api/images/%s' % (image))
        if resp.status != 200:
            log.info("Image slot does not yet exist.")
            raise RepomanError('Image does not yet exist.', resp)

        if owner:
            url = 'https://' + config.host + '/api/images/raw/%s/%s/%s' % (owner, hypervisor, image)
        else:
            url = 'https://' + config.host + '/api/images/raw/%s/%s' % (hypervisor, image)
        try:
            cmd = ['curl',
                    '--cert', config.proxy,
                    '--insecure',
                    '--head',
                    url]
            log.debug(" ".join(cmd))
            p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
            if not p:
                log.error("Error calling: %s" % (cmd))
                raise RepomanError("Error checking if image %s, owner %s, has an uploaded file for hypervisor %s" % (image, owner, hypervisor))
            log.info("Command complete")
            stdout = p.communicate()[0]
            log.debug(stdout)
            m = re.search('^HTTP/.+200 OK', stdout, flags=re.M)
            if m:
                log.info("Uploaded file exist for image %s, owner %s, hypervisor %s." % (image, owner, hypervisor))
                return True
            else:
                log.info("Uploaded file does not exist for image %s, owner %s, hypervisor %s." % (image, owner, hypervisor))
                return False
        except Exception, e:
            log.error("%s" % e)
            raise RepomanError(str(e))
Пример #14
0
    def create_image(self, imagepath, size=None):
        base_dir = os.path.dirname(imagepath)
        if not os.path.exists(base_dir):
            os.makedirs(base_dir)
            log.debug("Created image base dir: '%s'" % imagepath)
        
        root_stats = self.statvfs()
        snapshot_stats = self.statvfs(base_dir)
        
        # See if there is enough free space to create a snapshot (>50% free)
        if root_stats['used'] > snapshot_stats['free']:
            log.error("Not enought free space. (used:%s free:%s)" % 
                      (root_stats['used'], snapshot_stats['free']))
            raise ImageUtilError("ERROR.  not enough free space")    
        
        # If specified, see if the requested size is enough
        if size:
            if size < root_stats['used']:
                log.error("Specified Image size less then required")
                log.error("Required:%s  Requested:%s" % (root_stats['used'], size))
                raise ImageUtilError("Specified partition size is less then needed.")
        else:
            size = root_stats['size']            

        self.dd_sparse(imagepath, size)
        if self.partition:
            self.create_bootable_partition(imagepath)
            self.create_device_map(imagepath)
            label = self.get_fs_label('/')
            if label == None:
                raise ImageUtilError("Your VM is partitioned but the partition where / is mounted is not labeled.  Please see the repoman manpages for more information about the requirements for partitioned images.")
            self.mkfs(self.device_map, label=label, fs_type = self.detect_fs_type('/'))
            self.delete_device_map(imagepath)
        else:
            self.mkfs(imagepath, fs_type = self.detect_fs_type('/'))
Пример #15
0
    def snapshot_system(self, start_fresh=False, verbose=False, clean=False):
        if verbose:
            ch = logging.StreamHandler()
            ch.setLevel(logging.INFO)
            formatter = logging.Formatter("%(levelname)s - %(message)s")
            ch.setFormatter(formatter)
            log.addHandler(ch)

        log.debug("Obtaining lock")
        if not self.obtain_lock():
            log.error("Unable to obtain lock")
            raise ImageUtilError("Unable to obtain lock")
        snapshot_success = False
        try:
            self._snapshot_system(start_fresh, verbose, clean)
            snapshot_success = True
        finally:
            log.debug("Releasing lock")
            self.destroy_lock()
            log.info("Unmounting Image")
            self.umount_image()
        if self.partition and snapshot_success:
                self.install_mbr(self.imagepath)
Пример #16
0
 def _check_response(self, resp):
     if resp.status in [httplib.OK, httplib.CREATED]:
         return resp
     elif resp.status == httplib.BAD_REQUEST:
         # 400
         message = resp.read()
         # parse body for reason and display to user.
     elif resp.status == httplib.FORBIDDEN:
         # 403
         message = "You lack the rights to access that object"
     elif resp.status == httplib.NOT_FOUND:
         # 404
         message = "Requested resource could not be found"
     elif resp.status == httplib.REQUEST_TIMEOUT:
         # 408
         message = "Request has timed out.  Please retry or seek assistance."
     elif resp.status == httplib.CONFLICT:
         # 409
         message = "Conflict in request"
         # parse body for reason and display to user.
     elif resp.status == httplib.INTERNAL_SERVER_ERROR:
         # 500
         message = ("The server has encountered an error and was unable to "
                    "process your request.  If problem persists, seek assistance.")
     elif resp.status == httplib.NOT_IMPLEMENTED:
         # 501
         message = ("The requested functionality has yet to be implemented by the server")
     else:
         # Generic error message
         message = ("Response from server cannot be handled by this client.\n\n"
                    "status: %s\nreason: %s\n"
                    "-------- body ---------\n"
                    "%s\n-----------------------\n"
                    ) % (resp.status, resp.reason, resp.read())
     log.error(message)
     raise RepomanError(message, resp)
Пример #17
0
 def create_device_map(self, path):
     if self.device_map != None:
         log.error("Attempt to create device map over existing one.  Aborting.")
         raise ImageUtilError("Error creating device map.")
     cmd = ['kpartx', '-av' , path]
     log.debug("Creating device map for %s" % (path))
     log.debug(cmd)
     p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
     if not p:
         log.error("Error calling: %s" % (cmd))
         raise ImageUtilError("Error creating device map.")
     (stdout, stderr) = p.communicate()
     log.debug("[%s] kpartx output stdout:\n%s\n\nkpartx output stderr:\n%s" % (cmd, stdout, stderr))
     if p.returncode != 0:
         log.error("Device map creation command returned error: %d" % (p.returncode))
         raise ImageUtilError("Error creating device map.")
     # Search the output to extract the location of the new device map
     m = re.search('^add map (\w+) .+$', stdout, flags=re.M)
     if not m:
         log.error("Error extracting location of new device map from:\n%s" % (stdout))
         raise ImageUtilError("Error extracting location of new device map.")
     log.debug("Device map created for %s" % (path))
     self.device_map = '/dev/mapper/%s' % (m.group(1))
     return self.device_map
Пример #18
0
                kwargs['os_type'] = args.os_type
            if args.os_variant:
                kwargs['os_variant'] = args.os_variant

            self.write_metadata(kwargs, self.metadata_file)
        except IOError, e:
            log.error("Unable to write to root fs.")
            log.error("%s" % e)
            raise SubcommandFailure(self, "Could not write to %s, are you root?" % (self.metadata_file), e)
            
            
        try:
            print "Starting the snapshot process.  Please be patient, this will take a while."
            image_utils.snapshot_system(verbose=args.verbose, clean=args.clean)
        except ImageUtilError, e:
            log.error(e)
            raise SubcommandFailure(self, "An error occured during the snapshot process.", e)
            
        if not exists:
            try:
                image = self.get_repoman_client(args).create_image_metadata(**kwargs)
                print "[OK]    Creating image metadata on server."
            except RepomanError, e:
                log.error("Error while creating image slot on server")
                log.error(e)
                raise SubcommandFailure(self, "Error while creating image slot on server.", e)
        else:
            # Image exist.  Let's update it's metadata if needed
            image_name = kwargs['name']
            if args.owner:
                image_name = "%s/%s" % (args.owner, kwargs['name'])
Пример #19
0
                self._conn.request(method, url)
            elif method == 'DELETE':
                self._conn.request(method, url)
            elif method == 'POST':
                params = urllib.urlencode(kwargs)
                self._conn.request(method, url, params, headers)
            resp =  self._conn.getresponse()
            log.debug("Server response code: %s" % resp.status)
            return self._check_response(resp)
        except RepomanError, e:
            raise(e)
        except httplib.InvalidURL, e:
            log.error("%s" % e)
            raise RepomanError("Invalid port number")
        except httplib.HTTPException, e:
            log.error("%s" % e)
            print 'httpexception'
        except socket.gaierror, e:
            log.error("%s", e)
            raise RepomanError('Unable to connect to server.  Check Host and port \n\t\t %s' % e)
#        except socket.error, e:
#            raise RepomanError('Unable to connect to server.  Is the server running?\n\t%s' % e)
#        except ssl.SSLError, e:
#            pass
        except Exception, e:
            log.error("%s", e)
            if str(e).find('SSL_CTX_use_PrivateKey_file') and not os.path.exists(self.PROXY):
                raise ProxyNotFoundError('Certificate proxy not found: %s\nPlease create a certificate proxy and try again.' % (self.PROXY))
            elif str(e).find('certificate expired') != -1:
                raise ProxyExpiredError('Your certificate proxy has expired.\nPlease generate a new one and try again.')
            else: