def _set_modified_files(self, filelist):
"""
Populates self.modified with a list of all modified files, and stores
the file diff in self.changeset dictionary.
"""
log.debug("Finding modified files.")
for filename in filelist:
ltpath = os.path.join(self.pristinepath, filename)
rtpath = os.path.join(self.installedpath, filename)
# System should always have access to the pristine
if not zuputils.hasAccess(ltpath):
raise ZenUpInternalError("Error accessing: %s" % ltpath)
# Check the diff if the right side path is accessible, otherwise
# store the path in the unknown list
if zuputils.hasAccess(rtpath):
if os.path.isfile(rtpath):
with open(ltpath) as fp:
ltdata = self._adjusted_file(fp)
with open(rtpath) as fp:
rtdata = self._adjusted_file(fp)
diff = self.unified_diff(filename, ltdata, rtdata)
if diff:
log.debug("Found diff for file: %s", filename)
self.modified.append(filename)
self.changeset[filename] = diff
else:
log.debug("Found UNKNOWN file: %s", filename)
self.unknown.append(filename)
def _loadProducts(self):
status = os.path.join(self.HOME, self.STATUS_FILE)
if not self.HOME or \
not os.path.exists(self.HOME) or \
not os.path.isdir(self.HOME) or \
not zuputils.hasAccess(self.HOME):
raise ZenUpException("Unable to access zenup home: %s" % self.HOME)
elif not os.path.exists(status):
return
try:
with open(status) as fp:
productYaml = yaml.load_all(fp)
if not productYaml:
return
for productData in productYaml:
product = zupproduct.ZupProduct(**productData)
log.debug("Loaded product: %s", product.id_)
self.products[product.id_] = product
except Exception as e:
raise ZenUpException("Error loading products from %s: %s" %
(self.STATUS_FILE, e))
def _set_added_files(self, filelist):
"""
Populates self.added with a list of all added files that are
accessible (added files are files that exist on the installed path,
but not on the pristine path)
"""
for filename in filelist:
ltpath = os.path.join(self.pristinepath, filename)
rtpath = os.path.join(self.installedpath, filename)
# Open the file if the right side path is accessible, otherwise
# store the path in the unknown list
if zuputils.isSamePathType(ltpath, rtpath):
raise ZenUpInternalError("Error accessing: %s" % ltpath)
elif zuputils.hasAccess(rtpath):
if os.path.isfile(rtpath):
with open(rtpath) as fp:
self.changeset[filename] = self.unified_diff(
filename, [], self._adjusted_file(fp))
log.debug("Found added file: %s", filename)
self.added.append(filename)
else:
log.debug("Found UNKNOWN file: %s", filename)
self.unknown.append(filename)
def open(self):
try:
with tarfile.open(self.zupfile) as tar:
if hasattr(self, "path"):
if not self.path:
self.path = tempfile.mkdtemp(
prefix="%s_" % os.path.basename(self.zupfile))
elif not zuputils.hasAccess(os.path.dirname(self.path)):
raise ZupArchiveException("Cannot load zup at " \
"path: %s" % self.path)
elif not os.path.exists(self.path):
os.mkdir(self.path)
else:
path = self.path
delattr(self, "path")
raise ZupArchiveException("Path already exists: %s" % path)
tar.extractall(self.path)
self.product, self.revision, self.created = \
self._check_archive()
self.fixes, self.patches, self.changes = \
self._check_manifest()
else:
raise ZenUpInternalError("ZupArchive object not initialized")
except tarfile.TarError as e:
raise ZupArchiveException("Errors while accessing archive: %s" % self.zupfile)
except OSError as e:
raise ZupArchiveException("Unable to export archive at path: %s" % self.path)
except IOError as e:
raise ZupArchiveException("Archive not accessible: %s" % self.zupfile)
except ZupArchiveReadError as e:
self.close()
raise
return self
def _set_deleted_files(self, filelist):
"""
Populates self.deleted with a list of all deleted files that are
accessible (deleted files are files that exist on the pristine path,
but not on the installed path)
"""
opcodes = CHECK, DELETE, UNKNOWN = 0, 1, 2
# Filelist is sorted, so parent paths will always directly precede its
# children
filelist.sort()
i = 0
op = CHECK
while i < len(filelist):
# If the path is confirmed for deletion, mark the proceeding
# children as deleted
if DELETE == op:
if filelist[i].startswith(path):
ltpath = os.path.join(self.pristinepath, filelist[i])
self.deleted.append(filelist[i])
if os.path.isfile(ltpath):
with open(ltpath) as fp:
self.changeset[filelist[i]] = self.unified_diff(
filelist[i], self._adjusted_file(fp), [])
i += 1
else:
op = CHECK
# If the path is unknown, mark the proceeding children as unknown
elif UNKNOWN == op:
if filelist[i].startswith(parentpath):
self.unknown.append(filelist[i])
i += 1
else:
op = CHECK
# Determines if the path was deleted
if CHECK == op:
path = filelist[i]
parentpath = self._parent_path(self.installedpath, path)
rtpath = os.path.join(self.installedpath, path)
if self._is_binary(rtpath):
self.unknown.append(filelist[i])
i += 1
elif zuputils.hasAccess(
os.path.join(self.installedpath, parentpath)):
op = DELETE
else:
op = UNKNOWN
elif op not in opcodes:
raise ZenUpInternalError("Unknown opcode: %s" % op)
def __init__(self, pristinepath, installedpath):
"""
pristinepath: path to the pristine source (AKA the left side of the
diff)
installedpath: path to the installed source (AKA the right side of the
diff)
"""
if zuputils.hasAccess(pristinepath):
self.pristinepath = pristinepath
self.installedpath = installedpath
else:
raise ZenUpInternalError("Cannot access pristine: %s" \
% pristinepath)
self.changeset = {}
self.modified = []
self.added = []
self.deleted = []
self.unknown = []
self.verbose = True
def _dry_run(self, zupfile, runner, force=False):
log.debug("Initializing environment (product=%s, zupfile=%s)" %
(self.id_, zupfile))
output = None
# Extract the new zup and verify usability/compatibility
zup = runner.add_archive(zupfile, self._NEW_ZUP)
self._check_compatibility(zup)
runner.set_zup_product_dir(self.home)
runner.set_zup_working_dir(zup.path)
zup.check()
# Extract the pristine source
runner.add_source(self.getSourceArchPath())
source = os.path.join(runner.source, self.SOURCE_ARCH_SOURCE)
gen_patch_apply = zup.apply(source)
# Apply patches from the currently installed zup onto the pristine
if self.zupfile:
path = os.path.join(self.getPacksDirPath(), self.zupfile)
if not zuputils.hasAccess(path):
raise ZenUpInternalError("Unable to access product zup: %s" %
self.zupfile)
head_zup = runner.add_archive(path, self._HEAD_ZUP)
patch_count = len(head_zup.patches)
for i in xrange(patch_count):
gen_patch_apply.next()
else:
patch_count = 0
# Compile list of files affected by the new zup excluding files
# affected by the current zup
log.debug("Computing files affected by installing zup %s (%s)" %
(zupfile, self.id_))
affected_files = set()
for patch_file in zup.patches[patch_count:]:
if patch_file not in zup.changes:
raise ZenUpInternalError("Missing metadata for patch: %s" %
patch_file)
file_list = zup.changes[patch_file]
added = file_list.get("adds") or []
if isinstance(added, basestring):
affected_files.add(added)
else:
affected_files.update(added)
deleted = file_list.get("deletes") or []
if isinstance(deleted, basestring):
affected_files.add(deleted)
else:
affected_files.update(deleted)
modified = file_list.get("modifies") or []
if isinstance(modified, basestring):
affected_files.add(modified)
else:
affected_files.update(modified)
# Local diff
ld = localdiff.LocalDiff(source, self.home)
config = self._get_config(runner)
if not force:
_config = {
ld.INCLUDE_DIR:
set(os.path.dirname(f) for f in affected_files),
ld.INCLUDE_FILE: affected_files
}
# update our constructed config with the actual config from file.
_config.update(config)
config = _config
ld.run(**config)
# Files/Dirs should not be unknown
if ld.unknown:
errmsg = []
errmsg.append("Cannot patch the following path(s):")
errmsg.extend(["\t%s" % filename for filename in ld.unknown])
raise ZenUpProductException("\n".join(errmsg))
# Apply the remaining patches from the new zup onto the source
for i in gen_patch_apply:
pass
# Merge local changes
log.debug("Merging local diff (product=%s, zupfile=%s)" %
(self.id_, zupfile))
if force:
affected_files.update(ld.added)
affected_files.update(ld.deleted)
affected_files.update(ld.modified)
# Files/Directories that are deleted locally and are affected by the
# zup cannot be merged.
elif ld.deleted:
errmsg = []
errmsg.append("The following path(s) have been deleted and cannot "
"be patched:")
errmsg.extend(["\t%s" % filename for filename in ld.deleted])
raise ZenUpProductException("\n".join(errmsg))
# Try to merge any other local diffs
elif ld.changeset:
pwd = os.getcwd()
local_diff_file = os.path.join(runner.path, self._LOCAL_DIFF_FILE)
reject_file = os.path.join(runner.path, self._REJECT_FILE)
try:
os.chdir(source)
with open(local_diff_file, "w+") as fp:
fp.write(str(ld))
fp.seek(0)
options = ["-N", "-r%s" % reject_file, "-p0"]
try:
zuputils.applyPatch(fp, *options)
except ZenUpException as e:
FORWARD_PATCH_MESSAGE = "Reversed (or previously " \
"applied) patch detected! Skipping patch."
NEW_FILE_MESSAGE = "patching file"
if os.path.exists(reject_file):
with open(reject_file) as fp:
log.info("Merge Conflicts:")
[
log.info(line.rstrip('\n'))
for line in fp.readlines()
]
ignore = False
for line in str(e).splitlines():
if line.startswith(NEW_FILE_MESSAGE):
ignore = False
elif line == FORWARD_PATCH_MESSAGE:
ignore = True
elif line.rfind(reject_file) > 0:
if not ignore:
raise ZenUpProductException(
"Conflict(s) merging local diff:\n%s" %
str(e).replace(reject_file,
"zenup log file"))
output = str(e).replace(reject_file, "zenup log file")
finally:
os.chdir(pwd)
# Calculate the state of all files affected by the ZUP
files_added = []
files_deleted = []
files_modified = []
for filename in affected_files:
in_pristine = os.path.exists(os.path.join(source, filename))
in_installed = os.path.exists(os.path.join(self.home, filename))
is_dir = os.path.isdir(os.path.join(source, filename))
is_added = in_pristine and not in_installed
is_deleted = not in_pristine and in_installed
is_modified = in_pristine and in_installed and not is_dir
if is_added:
files_added.append(filename)
elif is_deleted:
files_deleted.append(filename)
elif is_modified:
files_modified.append(filename)
return files_added, files_deleted, files_modified, output
def patch(self, patchfile, message=None, options=None):
"""
Applies a single patch directly onto the product's home directory
while allowing the user to add their own individualized comments.
"""
message = message or ""
pwd = os.getcwd()
patchfile = os.path.join(pwd, patchfile)
if not zuputils.hasAccess(patchfile):
raise ZenUpException("Cannot access patch file %s" % patchfile)
if not zuputils.hasAccess(self.home):
raise ZenUpInternalError("Cannot access product home %s (%s)" %
(self.home, self.id_))
isStrip = False
if options:
ops = options.split()
for op in ops:
if op.startswith(("-p", "--strip")):
isStrip = True
break
else:
ops = []
if not isStrip:
ops.append("-p0")
try:
os.chdir(self.home)
with open(patchfile) as fp:
zuputils.applyPatch(fp, "--dry-run", *ops)
fp.seek(0)
filename = os.path.basename(patchfile)
directory = self.getPatchesDirPath()
fname, ext = os.path.splitext(
filename) # "hello_world", ".txt"
i = 1
while os.path.exists(os.path.join(directory, filename)):
i += 1
filename = "%s%s.%d" % (fname, ext, i)
if not os.path.exists(directory):
os.mkdir(directory)
path = os.path.join(directory, filename)
shutil.copyfile(patchfile, path)
zuputils.applyPatch(fp, *ops)
except ZenUpInternalError as e:
raise ZenUpException(e)
finally:
os.chdir(pwd)
with open(self.getMessageFilePath(), "a") as fp:
timestamp = datetime.now().strftime("%c")
if options:
fp.write('%s [%s "%s"] %s\n' %
(timestamp, filename, options, message))
else:
fp.write('%s [%s] %s\n' % (timestamp, filename, message))
self.lastupdate = timestamp
def install(self, source, path, displayOutput=False):
# Validation
log.debug("Performing validation to initialize product")
if displayOutput:
print "Validating..."
if self.id_ or os.path.exists(self.path):
raise ZenUpProductException("Product already exists: %s" \
% self.id_)
if not os.path.exists(path) or \
not zuputils.hasAccess(path):
raise ZenUpInternalError("Cannot create product at path: %s" \
% self.path)
if not os.path.exists(self.home):
raise ZenUpProductException("Cannot find product's home " \
"directory: %s" % self.home)
self.id_ = self.check_source(source)
self.name = self.name or self.id_
self.path = os.path.join(path, self.id_)
if os.path.exists(self.path):
raise ZenUpProductException("Product already exists: %s" \
% self.id_)
yield
# Installation
log.info("Initializing product: %s", self.id_)
if displayOutput:
print "Initializing..."
try:
# Create the zenup product installation path
os.mkdir(self.path)
with zuprunner.ZupRunner(self.id_) as runner:
# Unzip the pristine source into a temp directory
runner.add_source(source)
# Run the install script
install_script = os.path.join(runner.source,
self.INSTALL_SCRIPT)
zuputils.runScript(install_script)
# Write the contents of the pristine source into product's
# source archive file
with tarfile.open(self.getSourceArchPath(), "w:gz") as tar:
path_list = os.listdir(runner.source)
for path in path_list:
tar.add(os.path.join(runner.source, path), path)
# If the product is being initialized with a zup file...
if self.zupfile:
# Verify compatibility with the product and set the revision
# on the product
with zuparchive.ZupArchive(self.zupfile) as archive:
self._check_compatibility(archive)
self.revision = archive.revision
# Upload the pack to the product's pack directory
packname = "%s-SP%s.zup" % (self.id_, self.revision)
path = os.path.join(self.path, self.PACKS_DIR)
zupfile_orig = self.zupfile
self.zupfile = os.path.join(path, packname)
os.mkdir(path)
shutil.copy(zupfile_orig, self.zupfile)
else:
self.revision = ""
self.created = datetime.now().strftime("%c")
self.lastupdate = self.created
except OSError:
msg = "Error installing product: %s (%s)" % (self.id_, self.name)
log.exception(msg)
self.uninstall()
raise ZenUpProductException(msg)
except ZenUpException:
# ZEN-6864: QA didn't want an exception in the log when there isn't one in output.
log.error("Error installing product: %s" % self.id_)
self.uninstall()
raise
except Exception:
log.exception("Error installing product: %s" % self.id_)
self.uninstall()
raise
yield