def sync_fs(self, verbose):
#TODO: add progress bar into rsync somehow
log.info("Starting Sync Process")
exclude_list = ""
if self.system_excludes != None and len(self.system_excludes) > 0:
for exclude_item in self.system_excludes:
exclude_list += '--exclude "%s" ' % (exclude_item)
if self.user_excludes != None and len(self.user_excludes) > 0:
for exclude_item in self.user_excludes:
exclude_list += '--exclude "%s" ' % (exclude_item)
# Let's not forget to add the --delete-exclude flag to rsync else
# previously synced files which now match an exclude rule will not get
# deleted and will stay in the synced image.
if len(exclude_list) > 0:
exclude_list += " --delete-excluded"
flags = ''
if verbose:
flags += '--stats --progress '
cmd = "rsync -a --sparse %s --delete %s / %s" % (flags, exclude_list, self.mountpoint)
log.debug("%s" % cmd)
p = subprocess.Popen(cmd, shell=True, env=config.get_restricted_env()).wait()
if p:
log.error("Rsync encountered an issue. return code: '%s'" % p)
raise ImageUtilError("Rsync failed. Aborting.")
log.info("Sync Complete")
def get_fs_label(self, fs = '/'):
"""
Returns the given filesystem's label, or None if the filesystem has no label.
"""
log.debug("Detecting label for %s ..." % (fs))
# First detect the filesystem's partition.
partition = self.get_partition_for_fs(fs)
# Now use tune2fs to extract that partition's label
cmd = ['tune2fs', '-l', partition]
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
if not p:
log.error("Error calling: %s" % (cmd))
raise ImageUtilError("Error getting label for partition %s" % (partition))
stdout = p.communicate()[0]
log.debug("[%s] output:\n%s" % (cmd, stdout))
label = None
for l in stdout.split('\n'):
if l.startswith('Filesystem volume name'):
label = l.split(':')[1].strip()
if label == '<none>':
log.debug("Filesystem %s has no label." % (partition))
label = None
break
return label
def label_image(self, path, label='/'):
cmd = ['tune2fs', '-L', label, path]
log.debug("Labeling image: '%s'" % cmd)
if subprocess.Popen(cmd, shell=False, env=config.get_restricted_env()).wait():
log.error("Unable to label image")
raise ImageUtilError("Unable to label image")
log.debug("%s labeled as %s" % (path, label))
def dd_sparse(self, path, size_bytes):
cmd = ['dd', 'if=/dev/zero', 'of=%s' % path, 'count=0', 'bs=1', 'seek=%s' % size_bytes]
log.debug("Creating sparse file: '%s'" % cmd)
null_f = open('/dev/null', 'w')
if subprocess.Popen(cmd, shell=False, stdout=null_f, stderr=null_f, env=config.get_restricted_env()).wait():
log.error("Unable to create sparse file")
raise ImageUtilError("Error creating sparse file")
null_f.close()
def upload_image(self, image, owner, image_file, gzip=False, hypervisor='xen'):
log.info("Checking to see if image slot exists on repository")
if owner:
resp = self._get('/api/images/%s/%s' % (owner, image))
else:
resp = self._get('/api/images/%s' % (image))
if resp.status != 200:
log.info("Image slot does not yet exist.")
raise RepomanError('Image does not yet exist. Create an image before uploading to it', resp)
# Check if the source is a directory. If it is, then raise an
# exception.
if os.path.isdir(image_file):
raise RepomanError('Specified source is a directory: %s\nSource must be a file.' % (image_file))
# Check if the source file exists.
if not os.path.exists(image_file):
raise RepomanError('Specified source not found: %s' % (image_file))
if owner:
url = 'https://' + config.host + '/api/images/raw/%s/%s/%s' % (owner, hypervisor, image)
else:
url = 'https://' + config.host + '/api/images/raw/%s/%s' % (hypervisor, image)
try:
if gzip:
log.info("Performing gzip on image prior to upload")
print "Gzipping image before upload"
gzip_image = os.path.join(os.path.dirname(image_file), image)
gzip = subprocess.Popen("gzip --stdout %s > %s" % (image_file, gzip_image),
shell=True, env=config.get_restricted_env())
gzip.wait()
image_file = gzip_image
log.info('Gzip complete')
args = ['curl',
'--cert', config.proxy,
'--insecure',
'-T', image_file, url]
cmd = " ".join(args)
log.info("Running command: '%s'" % cmd)
curl = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, env=config.get_restricted_env())
for line in curl.stdout.readlines():
print line
log.info("Command complete")
# Cleanup gzip file if needed.
if gzip:
try:
log.info("Cleaning up %s" % (image_file))
os.remove(image_file)
except Exception, e:
pass
except Exception, e:
log.error("%s" % e)
raise RepomanError(str(e))
def mkfs(self, path, fs_type='ext3', label='/'):
if fs_type == None:
fs_type = 'ext3' # Default to ext3 if autodetection failed.
cmd = ['mkfs', '-t', fs_type, '-I', '128', '-F', '-L', label, path]
log.debug("Creating file system: '%s'" % cmd)
null_f = open('/dev/null', 'w')
if subprocess.Popen(cmd, shell=False, stdout=null_f, stderr=null_f, env=config.get_restricted_env()).wait():
log.error("Unable to create filesystem")
raise ImageUtilError("Error creating filesystem.")
null_f.close()
def delete_device_map(self, path):
cmd = ['kpartx', '-d', path]
log.debug("Deleting device map for %s" % (path))
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
if not p:
log.error("Error calling: %s" % (cmd))
raise ImageUtilError("Error deleting device map.")
stdout = p.communicate()[0]
log.debug("[%s] output:\n%s" % (cmd, stdout))
if p.returncode != 0:
log.error("Error deleting device map for %s" % (path))
raise ImageUtilError("Error deleting device map.")
log.debug("Device map deleted for %s" % (path))
self.device_map = None
def create_bootable_partition(self, path):
cmd = ['sfdisk', path]
log.debug("Creating bootable partition on %s" % (path))
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
if not p:
log.error("Error calling: %s" % (cmd))
raise ImageUtilError("Error creating bootable partition.")
p.stdin.write(',,L,*\n')
p.stdin.close()
p.wait()
if p.returncode != 0:
log.error("Command to create bootable partition returned error: %d" % (p.returncode))
raise ImageUtilError("Error creating bootable partition.")
log.debug("Bootable partition created on %s" % (path))
def get_partition_for_fs(self, fs):
"""
Returns the disk for a given fs.
"""
cmd = ['df', fs]
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
if not p:
log.error("Error calling: %s" % (cmd))
raise ImageUtilError("Error getting partition for filesystem %s" % (fs))
stdout = p.communicate()[0]
log.debug("[%s] output:\n%s" % (cmd, stdout))
feilds = stdout.split('\n')[1].split()
log.debug("Filesystem %s is on partition %s" % (fs, feilds[0]))
return feilds[0]
def install_mbr(self, path):
cmd = ['grub']
log.debug("Creating MBR on %s" % (path))
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
if not p:
log.error("Error calling: %s" % (cmd))
raise ImageUtilError("Error creating MBR on %s." % (path))
p.stdin.write('device (hd0) %s\n' % (path))
p.stdin.write('root (hd0,0)\n')
p.stdin.write('setup (hd0)\n')
p.stdin.write('quit\n')
p.stdin.close()
p.wait()
if p.returncode == 0:
log.debug("MBR created on %s" % (path))
else:
log.error("Error creating MBR on %s\nReturn code: %d" % (path, p.returncode))
raise ImageUtilError("Error creating MBR.")
def __call__(self, args):
# Check if sudo...
if os.getuid() != 0:
raise SubcommandFailure(self, "Error. This command requires root privileges, try again with sudo.")
kwargs={}
name = args.image
# Check for proper Gzip extension (if needed)
if args.gzip:
if name and name.endswith('.gz'):
pass
else:
log.info("Enforcing '.gz' extension.")
name += '.gz'
print ("WARNING: gzip option found, but your image name does not"
" end in '.gz'. Modifying image name to enforce this.")
print "New image name: '%s'" % (name)
kwargs['name'] = name
# Check to see if the image name has the '.gz' suffix and
# compression not enabled. Having uncompressed images named with
# a '.gz' suffix will break things in Nimbus.
if (not args.gzip) and name.endswith('.gz'):
print ("WARNING: The image name you gave ends with '.gz' but you did not enable image compression with the --gzip command line argument. Having uncompressed images with a '.gz' suffix to the image name can cause problems in some cloud frameworks.")
if not yes_or_no('Do you want to continue? [yes]/[n]o: '):
print "Aborting. Please select a new image name or enable compression via the --gzip command line argument."
return
exists = False
try:
image = self.get_repoman_client(args).describe_image(name, args.owner)
if image:
log.info("Found existing image")
exists = True
except RepomanError,e:
if e.status == 404:
log.debug("Did not find an existing image in repository with same name.")
pass
else:
log.error("Unexpected response occurred when testing if image exists.")
log.error("%s" % e)
raise SubcommandFailure(self, "Unexpected response from server occurred when testing if image exists.", e)
def is_disk_partitioned(self):
"""
Detects if the disk is partitioned or not.
Returns True if the disk is partitioned, False otherwise.
"""
cmd = ['df', '/']
log.debug("Checking if disk is partitioned...")
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
if not p:
log.error("Error calling: %s" % (cmd))
raise ImageUtilError("Error checking if disk is partitioned.")
stdout = p.communicate()[0]
log.debug("[%s] output:\n%s" % (cmd, stdout))
feilds = stdout.split('\n')[1].split()
if feilds[0][-1].isdigit():
log.debug('Disk is partitioned.')
return True
else:
log.debug('Disk is not partitioned.')
return False
def uploaded_image_exist(self, image, owner, hypervisor='xen'):
"""
This method will test if an image has an uploaded file for a given hypervisor.
"""
log.info("Checking to see if image %s, owner %s, has an uploaded file for hypervisor %s" % (image, owner, hypervisor))
if owner:
resp = self._get('/api/images/%s/%s' % (owner, image))
else:
resp = self._get('/api/images/%s' % (image))
if resp.status != 200:
log.info("Image slot does not yet exist.")
raise RepomanError('Image does not yet exist.', resp)
if owner:
url = 'https://' + config.host + '/api/images/raw/%s/%s/%s' % (owner, hypervisor, image)
else:
url = 'https://' + config.host + '/api/images/raw/%s/%s' % (hypervisor, image)
try:
cmd = ['curl',
'--cert', config.proxy,
'--insecure',
'--head',
url]
log.debug(" ".join(cmd))
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
if not p:
log.error("Error calling: %s" % (cmd))
raise RepomanError("Error checking if image %s, owner %s, has an uploaded file for hypervisor %s" % (image, owner, hypervisor))
log.info("Command complete")
stdout = p.communicate()[0]
log.debug(stdout)
m = re.search('^HTTP/.+200 OK', stdout, flags=re.M)
if m:
log.info("Uploaded file exist for image %s, owner %s, hypervisor %s." % (image, owner, hypervisor))
return True
else:
log.info("Uploaded file does not exist for image %s, owner %s, hypervisor %s." % (image, owner, hypervisor))
return False
except Exception, e:
log.error("%s" % e)
raise RepomanError(str(e))
def create_image(self, imagepath, size=None):
base_dir = os.path.dirname(imagepath)
if not os.path.exists(base_dir):
os.makedirs(base_dir)
log.debug("Created image base dir: '%s'" % imagepath)
root_stats = self.statvfs()
snapshot_stats = self.statvfs(base_dir)
# See if there is enough free space to create a snapshot (>50% free)
if root_stats['used'] > snapshot_stats['free']:
log.error("Not enought free space. (used:%s free:%s)" %
(root_stats['used'], snapshot_stats['free']))
raise ImageUtilError("ERROR. not enough free space")
# If specified, see if the requested size is enough
if size:
if size < root_stats['used']:
log.error("Specified Image size less then required")
log.error("Required:%s Requested:%s" % (root_stats['used'], size))
raise ImageUtilError("Specified partition size is less then needed.")
else:
size = root_stats['size']
self.dd_sparse(imagepath, size)
if self.partition:
self.create_bootable_partition(imagepath)
self.create_device_map(imagepath)
label = self.get_fs_label('/')
if label == None:
raise ImageUtilError("Your VM is partitioned but the partition where / is mounted is not labeled. Please see the repoman manpages for more information about the requirements for partitioned images.")
self.mkfs(self.device_map, label=label, fs_type = self.detect_fs_type('/'))
self.delete_device_map(imagepath)
else:
self.mkfs(imagepath, fs_type = self.detect_fs_type('/'))
def snapshot_system(self, start_fresh=False, verbose=False, clean=False):
if verbose:
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
formatter = logging.Formatter("%(levelname)s - %(message)s")
ch.setFormatter(formatter)
log.addHandler(ch)
log.debug("Obtaining lock")
if not self.obtain_lock():
log.error("Unable to obtain lock")
raise ImageUtilError("Unable to obtain lock")
snapshot_success = False
try:
self._snapshot_system(start_fresh, verbose, clean)
snapshot_success = True
finally:
log.debug("Releasing lock")
self.destroy_lock()
log.info("Unmounting Image")
self.umount_image()
if self.partition and snapshot_success:
self.install_mbr(self.imagepath)
def _check_response(self, resp):
if resp.status in [httplib.OK, httplib.CREATED]:
return resp
elif resp.status == httplib.BAD_REQUEST:
# 400
message = resp.read()
# parse body for reason and display to user.
elif resp.status == httplib.FORBIDDEN:
# 403
message = "You lack the rights to access that object"
elif resp.status == httplib.NOT_FOUND:
# 404
message = "Requested resource could not be found"
elif resp.status == httplib.REQUEST_TIMEOUT:
# 408
message = "Request has timed out. Please retry or seek assistance."
elif resp.status == httplib.CONFLICT:
# 409
message = "Conflict in request"
# parse body for reason and display to user.
elif resp.status == httplib.INTERNAL_SERVER_ERROR:
# 500
message = ("The server has encountered an error and was unable to "
"process your request. If problem persists, seek assistance.")
elif resp.status == httplib.NOT_IMPLEMENTED:
# 501
message = ("The requested functionality has yet to be implemented by the server")
else:
# Generic error message
message = ("Response from server cannot be handled by this client.\n\n"
"status: %s\nreason: %s\n"
"-------- body ---------\n"
"%s\n-----------------------\n"
) % (resp.status, resp.reason, resp.read())
log.error(message)
raise RepomanError(message, resp)
def create_device_map(self, path):
if self.device_map != None:
log.error("Attempt to create device map over existing one. Aborting.")
raise ImageUtilError("Error creating device map.")
cmd = ['kpartx', '-av' , path]
log.debug("Creating device map for %s" % (path))
log.debug(cmd)
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=config.get_restricted_env())
if not p:
log.error("Error calling: %s" % (cmd))
raise ImageUtilError("Error creating device map.")
(stdout, stderr) = p.communicate()
log.debug("[%s] kpartx output stdout:\n%s\n\nkpartx output stderr:\n%s" % (cmd, stdout, stderr))
if p.returncode != 0:
log.error("Device map creation command returned error: %d" % (p.returncode))
raise ImageUtilError("Error creating device map.")
# Search the output to extract the location of the new device map
m = re.search('^add map (\w+) .+$', stdout, flags=re.M)
if not m:
log.error("Error extracting location of new device map from:\n%s" % (stdout))
raise ImageUtilError("Error extracting location of new device map.")
log.debug("Device map created for %s" % (path))
self.device_map = '/dev/mapper/%s' % (m.group(1))
return self.device_map
kwargs['os_type'] = args.os_type
if args.os_variant:
kwargs['os_variant'] = args.os_variant
self.write_metadata(kwargs, self.metadata_file)
except IOError, e:
log.error("Unable to write to root fs.")
log.error("%s" % e)
raise SubcommandFailure(self, "Could not write to %s, are you root?" % (self.metadata_file), e)
try:
print "Starting the snapshot process. Please be patient, this will take a while."
image_utils.snapshot_system(verbose=args.verbose, clean=args.clean)
except ImageUtilError, e:
log.error(e)
raise SubcommandFailure(self, "An error occured during the snapshot process.", e)
if not exists:
try:
image = self.get_repoman_client(args).create_image_metadata(**kwargs)
print "[OK] Creating image metadata on server."
except RepomanError, e:
log.error("Error while creating image slot on server")
log.error(e)
raise SubcommandFailure(self, "Error while creating image slot on server.", e)
else:
# Image exist. Let's update it's metadata if needed
image_name = kwargs['name']
if args.owner:
image_name = "%s/%s" % (args.owner, kwargs['name'])
self._conn.request(method, url)
elif method == 'DELETE':
self._conn.request(method, url)
elif method == 'POST':
params = urllib.urlencode(kwargs)
self._conn.request(method, url, params, headers)
resp = self._conn.getresponse()
log.debug("Server response code: %s" % resp.status)
return self._check_response(resp)
except RepomanError, e:
raise(e)
except httplib.InvalidURL, e:
log.error("%s" % e)
raise RepomanError("Invalid port number")
except httplib.HTTPException, e:
log.error("%s" % e)
print 'httpexception'
except socket.gaierror, e:
log.error("%s", e)
raise RepomanError('Unable to connect to server. Check Host and port \n\t\t %s' % e)
# except socket.error, e:
# raise RepomanError('Unable to connect to server. Is the server running?\n\t%s' % e)
# except ssl.SSLError, e:
# pass
except Exception, e:
log.error("%s", e)
if str(e).find('SSL_CTX_use_PrivateKey_file') and not os.path.exists(self.PROXY):
raise ProxyNotFoundError('Certificate proxy not found: %s\nPlease create a certificate proxy and try again.' % (self.PROXY))
elif str(e).find('certificate expired') != -1:
raise ProxyExpiredError('Your certificate proxy has expired.\nPlease generate a new one and try again.')
else: