def initialize(self):
# Open the database.
self.open()
# Check if we actually can open the database.
if not self.format in DATABASE_FORMATS_SUPPORTED:
raise DatabaseFormatError, _("The format of the database is not supported by this version of pakfire.")
def __init__(self, pakfire, repo, filename):
base.Package.__init__(self, pakfire, repo)
self.filename = os.path.abspath(filename)
# Place to cache the metadata
self._metadata = {}
# Place to cache the filelist and payload compression algorithm.
self._filelist = None
self.__payload_compression = None
# Store the format of this package file.
self.format = self.get_format()
# XXX need to make this much better.
self.check()
# Read the info file.
if self.format >= 1:
a = self.open_archive()
f = a.extractfile("info")
self.lexer = lexer.FileLexer(f.readlines())
f.close()
a.close()
elif self.format == 0:
pass
else:
raise PackageFormatUnsupportedError, _("Filename: %s") % self.filename
def update_metadata(self, force=False, offline=False):
filename = os.path.join(METADATA_DOWNLOAD_PATH, METADATA_DOWNLOAD_FILE)
cache_filename = self.cache_path(os.path.basename(filename))
# Check if the metadata is already recent enough...
exists = self.cache.exists(cache_filename)
if not exists and offline:
raise OfflineModeError, _("No metadata available for repository %s. Cannot download any.") \
% self.name
elif exists and offline:
# Repository metadata exists. We cannot update anything because of the offline mode.
return
if not force and exists:
age = self.cache.age(cache_filename)
if age and age < TIME_10M:
log.debug(
"Metadata is recent enough. I don't download it again.")
return
# Going to download metada.
log.debug("Going to download repository metadata for %s..." %
self.name)
assert not offline
grabber = downloader.MetadataDownloader(self.pakfire)
grabber = self.mirrors.group(grabber)
while True:
try:
data = grabber.urlread(filename, limit=METADATA_DOWNLOAD_LIMIT)
except urlgrabber.grabber.URLGrabError, e:
if e.errno == 256:
raise DownloadError, _(
"Could not update metadata for %s from any mirror server"
) % self.name
grabber.increment_mirror(grabber)
continue
# Parse new metadata for comparison.
md = metadata.Metadata(self.pakfire, metadata=data)
if self.metadata and md < self.metadata:
log.warning(
_("The downloaded metadata was less recent than the current one."
))
grabber.increment_mirror(grabber)
continue
# If the download went well, we write the downloaded data to disk
# and break the loop.
f = self.cache.open(cache_filename, "w")
f.write(data)
f.close()
break
def update_database(self, force=False, offline=False):
assert self.metadata, "Metadata needs to be openend first."
# Construct cache and download filename.
filename = os.path.join(METADATA_DOWNLOAD_PATH, self.metadata.database)
cache_filename = self.cache_path("database", self.metadata.database)
if not force:
force = not self.cache.exists(cache_filename)
# Raise an exception when we are running in offline mode but an update is required.
if force and offline:
raise OfflineModeError, _(
"Cannot download package database for %s in offline mode."
) % self.name
elif not force:
return
# Just make sure we don't try to download anything in offline mode.
assert not offline
# Initialize a grabber for download.
grabber = downloader.DatabaseDownloader(
self.pakfire,
text=_("%s: package database") % self.name,
)
grabber = self.mirrors.group(grabber)
while True:
# Open file on server.
urlobj = fileobj = grabber.urlopen(filename)
if self.metadata.database_compression:
fileobj = compress.decompressobj(
fileobj=fileobj, algo=self.metadata.database_compression)
# Make a new file in the cache.
cacheobj = self.cache.open(cache_filename, "wb")
try:
while True:
buf = fileobj.read(BUFFER_SIZE)
if not buf:
break
cacheobj.write(buf)
finally:
# XXX we should catch decompression errors
# Close all file descriptors.
cacheobj.close()
fileobj.close()
if not urlobj == fileobj:
urlobj.close()
break
def initialize(self):
# Open the database.
self.open()
# Check if we actually can open the database.
if not self.format in DATABASE_FORMATS_SUPPORTED:
raise DatabaseFormatError, _(
"The format of the database is not supported by this version of pakfire."
)
def execute(self):
# Save start time.
self.time_start = time.time()
if self.logger:
self.logger.debug(_("Executing command: %s in %s") % (self.command, self.chroot_path or "/"))
child = None
try:
# Create new child process
child = self.create_subprocess()
# Record the output.
self.tee_log(child)
except:
# In case there has been an error, kill children if they aren't done
if child and child.returncode is None:
os.killpg(child.pid, 9)
try:
if child:
os.waitpid(child.pid, 0)
except:
pass
# Raise original exception.
raise
finally:
# Save end time.
self.time_end = time.time()
# wait until child is done, kill it if it passes timeout
nice_exit = True
while child.poll() is None:
if self.timeout_has_been_exceeded():
nice_exit = False
os.killpg(child.pid, 15)
if self.timeout_has_been_exceeded(3):
nice_exit = False
os.killpg(child.pid, 9)
if not nice_exit:
raise commandTimeoutExpired, (_("Command exceeded timeout (%(timeout)d): %(command)s") % (self.timeout, self.command))
# Save exitcode.
self.exitcode = child.returncode
if self.logger:
self.logger.debug(_("Child returncode was: %s") % self.exitcode)
if self.exitcode and self.log_errors:
raise ShellEnvironmentError, (_("Command failed: %s") % self.command, self.exitcode)
return self.exitcode
def update_database(self, force=False, offline=False):
assert self.metadata, "Metadata needs to be openend first."
# Construct cache and download filename.
filename = os.path.join(METADATA_DOWNLOAD_PATH, self.metadata.database)
cache_filename = self.cache_path("database", self.metadata.database)
if not force:
force = not self.cache.exists(cache_filename)
# Raise an exception when we are running in offline mode but an update is required.
if force and offline:
raise OfflineModeError, _("Cannot download package database for %s in offline mode.") % self.name
elif not force:
return
# Just make sure we don't try to download anything in offline mode.
assert not offline
# Initialize a grabber for download.
grabber = downloader.DatabaseDownloader(
self.pakfire,
text = _("%s: package database") % self.name,
)
grabber = self.mirrors.group(grabber)
while True:
# Open file on server.
urlobj = fileobj = grabber.urlopen(filename)
if self.metadata.database_compression:
fileobj = compress.decompressobj(fileobj=fileobj,
algo=self.metadata.database_compression)
# Make a new file in the cache.
cacheobj = self.cache.open(cache_filename, "wb")
try:
while True:
buf = fileobj.read(BUFFER_SIZE)
if not buf:
break
cacheobj.write(buf)
finally:
# XXX we should catch decompression errors
# Close all file descriptors.
cacheobj.close()
fileobj.close()
if not urlobj == fileobj:
urlobj.close()
break
def lint(self):
errors = []
if not self.name:
errors.append(_("Package name is undefined."))
if not self.version:
errors.append(_("Package version is undefined."))
# XXX to do...
return errors
def lint(self):
errors = []
if not self.name:
errors.append(_("Package name is undefined."))
if not self.version:
errors.append(_("Package version is undefined."))
# XXX to do...
return errors
def update_metadata(self, force=False, offline=False):
filename = os.path.join(METADATA_DOWNLOAD_PATH, METADATA_DOWNLOAD_FILE)
cache_filename = self.cache_path(os.path.basename(filename))
# Check if the metadata is already recent enough...
exists = self.cache.exists(cache_filename)
if not exists and offline:
raise OfflineModeError, _("No metadata available for repository %s. Cannot download any.") \
% self.name
elif exists and offline:
# Repository metadata exists. We cannot update anything because of the offline mode.
return
if not force and exists:
age = self.cache.age(cache_filename)
if age and age < TIME_10M:
log.debug("Metadata is recent enough. I don't download it again.")
return
# Going to download metada.
log.debug("Going to download repository metadata for %s..." % self.name)
assert not offline
grabber = downloader.MetadataDownloader(self.pakfire)
grabber = self.mirrors.group(grabber)
while True:
try:
data = grabber.urlread(filename, limit=METADATA_DOWNLOAD_LIMIT)
except urlgrabber.grabber.URLGrabError, e:
if e.errno == 256:
raise DownloadError, _("Could not update metadata for %s from any mirror server") % self.name
grabber.increment_mirror(grabber)
continue
# Parse new metadata for comparison.
md = metadata.Metadata(self.pakfire, metadata=data)
if self.metadata and md < self.metadata:
log.warning(_("The downloaded metadata was less recent than the current one."))
grabber.increment_mirror(grabber)
continue
# If the download went well, we write the downloaded data to disk
# and break the loop.
f = self.cache.open(cache_filename, "w")
f.write(data)
f.close()
break
def terminate_worker(self, worker):
"""
Terminates the given worker.
"""
log.warning(_("Terminating worker process: %s") % worker)
worker.terminate()
def add_packages(self, files):
# Search for possible package files in the paths.
files = self.search_files(*files)
# Give up if there are no files to process.
if not files:
return
# Create progress bar.
pb = util.make_progress(_("%s: Adding packages...") % self.name, len(files))
i = 0
for file in files:
if pb:
i += 1
pb.update(i)
# Add the package to the repository.
self.add_package(file, optimize_index=False)
# Optimize the index.
self.optimize_index()
if pb:
pb.finish()
# Optimize the index.
self.index.optimize()
def run(self, resultdir):
# Create resultdir if it does not exist yet.
if not os.path.exists(resultdir):
os.makedirs(resultdir)
log.info(_("Building source package %s:") % self.pkg.package_filename)
# The filename where this source package is saved at.
target_filename = os.path.join(resultdir, self.pkg.package_filename)
# Add datafile to package.
datafile = self.create_datafile()
# Create filelist out of data.
filelist = self.create_filelist(datafile)
# Create metadata.
metafile = self.create_metafile(datafile)
# Add files to the tar archive in correct order.
self.add(metafile, "info")
self.add(filelist, "filelist")
self.add(datafile, "data.img")
# Build the final tarball.
try:
self.save(target_filename)
except:
# Remove the target file when anything went wrong.
os.unlink(target_filename)
raise
return target_filename
def open(self):
# Initialize database.
self.db.initialize()
# Create a progressbar.
pb = util.make_progress(_("Loading installed packages"), len(self.db))
# Remove all data from the current index.
self.index.clear()
i = 0
for pkg in self.db.packages:
if pb:
i += 1
pb.update(i)
self.index.add_package(pkg)
self.index.optimize()
if pb:
pb.finish()
# Mark repo as open.
self.opened = True
def cpu_model(self):
# Determine CPU model
cpuinfo = {}
with open("/proc/cpuinfo") as f:
for line in f.readlines():
# Break at an empty line, because all information after that
# is redundant.
if not line:
break
try:
key, value = line.split(":")
except:
pass # Skip invalid lines
key, value = key.strip(), value.strip()
cpuinfo[key] = value
ret = None
if self.uname.startswith("arm"):
try:
ret = "%(Hardware)s - %(Processor)s" % cpuinfo
except KeyError:
pass
else:
ret = cpuinfo.get("model name", None)
return ret or _("Could not be determined")
def open(self):
# Initialize database.
self.db.initialize()
# Create a progressbar.
pb = util.make_progress(_("Loading installed packages"), len(self.db))
# Remove all data from the current index.
self.index.clear()
i = 0
for pkg in self.db.packages:
if pb:
i += 1
pb.update(i)
self.index.add_package(pkg)
self.index.optimize()
if pb:
pb.finish()
# Mark repo as open.
self.opened = True
def add_packages(self, files):
# Search for possible package files in the paths.
files = self.search_files(*files)
# Give up if there are no files to process.
if not files:
return
# Create progress bar.
pb = util.make_progress(
_("%s: Adding packages...") % self.name, len(files))
i = 0
for file in files:
if pb:
i += 1
pb.update(i)
# Add the package to the repository.
self.add_package(file, optimize_index=False)
# Optimize the index.
self.optimize_index()
if pb:
pb.finish()
# Optimize the index.
self.index.optimize()
def download(self, files):
existant_files = []
download_files = []
for file in files:
filename = os.path.join(SOURCE_CACHE_DIR, file)
log.debug("Checking existance of %s..." % filename)
if os.path.exists(filename) and os.path.getsize(filename):
log.debug("...exists!")
existant_files.append(filename)
else:
log.debug("...does not exist!")
download_files.append(filename)
if download_files:
log.info(_("Downloading source files:"))
if self.pakfire.offline:
raise OfflineModeError, _("Cannot download source code in offline mode.")
# Create source download directory.
if not os.path.exists(SOURCE_CACHE_DIR):
os.makedirs(SOURCE_CACHE_DIR)
for filename in download_files:
try:
self.grabber.urlgrab(os.path.basename(filename), filename=filename)
except URLGrabError, e:
# Remove partly downloaded file.
try:
os.unlink(filename)
except OSError:
pass
raise DownloadError, "%s %s" % (os.path.basename(filename), e)
# Check if the downloaded file was empty.
if os.path.getsize(filename) == 0:
# Remove the file and raise an error.
os.unlink(filename)
raise DownloadError, _("Downloaded empty file: %s") \
% os.path.basename(filename)
log.info("")
def build_date(self):
"""
Automatically convert the UNIX timestamp from self.build_time to
a humanly readable format.
"""
if self.build_time is None:
return _("Not set")
return "%s UTC" % datetime.datetime.utcfromtimestamp(self.build_time)
def shutdown(self):
"""
Terminates all workers and exists the daemon.
"""
if not self.__running:
return
log.info(_("Shutting down..."))
self.__running = False
def filter_deps(deps, filters):
if not filters:
return deps
_filters = []
filtered_deps = []
# Compile all filters.
for filter in filters.splitlines():
# Convert to raw string to make escaping characters
# easy to the user.
try:
filter = "%r" % filter
_filter = re.compile(filter[1:-1])
except re.error:
log.warning(
_("Regular experession is invalid and has been skipped: %s"
) % filter)
continue
_filters.append(_filter)
filters = _filters
for dep in deps:
filtered = False
for filter in filters:
# Search for a match anywhere in the line.
m = re.search(filter, dep)
if not m:
continue
# Let the user know what has been done.
log.info(_("Filter '%(pattern)s' filtered %(dep)s.") % \
{ "pattern" : filter.pattern, "dep" : dep })
# Yes, we found a match.
filtered = True
break
if not filtered:
filtered_deps.append(dep)
return filtered_deps
def filter_deps(deps, filters):
if not filters:
return deps
_filters = []
filtered_deps = []
# Compile all filters.
for filter in filters.splitlines():
# Convert to raw string to make escaping characters
# easy to the user.
try:
filter = "%r" % filter
_filter = re.compile(filter[1:-1])
except re.error:
log.warning(_("Regular experession is invalid and has been skipped: %s") % filter)
continue
_filters.append(_filter)
filters = _filters
for dep in deps:
filtered = False
for filter in filters:
# Search for a match anywhere in the line.
m = re.search(filter, dep)
if not m:
continue
# Let the user know what has been done.
log.info(_("Filter '%(pattern)s' filtered %(dep)s.") % \
{ "pattern" : filter.pattern, "dep" : dep })
# Yes, we found a match.
filtered = True
break
if not filtered:
filtered_deps.append(dep)
return filtered_deps
def initialize(self):
# Nothing to do, if everything is already up to date.
if self.initialized:
return
log.info(_("Initializing repositories..."))
for repo in self:
repo.open()
# Empty line.
log.info("")
def parse_inherit(self):
line = self.get_line(self._lineno)
m = re.match(LEXER_PACKAGE_INHERIT, line)
if not m:
raise LexerError, "Not a template inheritance: %s" % line
self._lineno += 1
self._template = m.group(1)
# Check if template exists.
if not self.template:
log.warning(_("Template does not exist: %s") % self._template)
def extract(self, member, path=""):
target = os.path.join(path, member.name)
# Remove symlink targets, because tarfile cannot replace them.
if member.issym():
try:
os.unlink(target)
except OSError:
pass
# Extract file the normal way...
try:
tarfile.TarFile.extract(self, member, path)
except OSError, e:
log.warning(_("Could not extract file: /%(src)s - %(dst)s") \
% { "src" : member.name, "dst" : e, })
def extract(self, member, path=""):
target = os.path.join(path, member.name)
# Remove symlink targets, because tarfile cannot replace them.
if member.issym():
try:
os.unlink(target)
except OSError:
pass
# Extract file the normal way...
try:
tarfile.TarFile.extract(self, member, path)
except OSError, e:
log.warning(_("Could not extract file: /%(src)s - %(dst)s") \
% { "src" : member.name, "dst" : e, })
def track_dependencies(self, builder, path):
# Build filelist with all files that have been installed.
f = tempfile.NamedTemporaryFile(mode="w", delete=False)
filelist = f.name
try:
for dir, subdirs, files in os.walk(path):
f.write("%s\n" % dir)
for file in files:
file = os.path.join(dir, file)
f.write("%s\n" % file)
file = "/%s" % os.path.relpath(file, path)
self._filelist.add(file)
f.close()
log.info(
_("Searching for automatic dependencies for %s...") %
self.friendly_name)
# Search for provides.
res = builder.run_script("find-provides", path, filelist)
provides = set(res.splitlines())
# Search for requires.
res = builder.run_script("find-requires", path, filelist)
requires = set(res.splitlines()) - provides
finally:
os.unlink(filelist)
self._dependencies["provides"] = provides
self._dependencies["requires"] = requires
# Filter dependencies.
for key in ("prerequires", "requires", "provides", "conflicts",
"obsoletes"):
# Make sure this is a list.
try:
self._dependencies[key] = list(self._dependencies[key])
except KeyError:
self._dependencies[key] = []
# Filter out unwanted elements.
self._dependencies[key] = self.filter_deps(
self._dependencies[key], self.lexer.get_var("filter_%s" % key))
def track_dependencies(self, builder, path):
# Build filelist with all files that have been installed.
f = tempfile.NamedTemporaryFile(mode="w", delete=False)
filelist = f.name
try:
for dir, subdirs, files in os.walk(path):
f.write("%s\n" % dir)
for file in files:
file = os.path.join(dir, file)
f.write("%s\n" % file)
file = "/%s" % os.path.relpath(file, path)
self._filelist.add(file)
f.close()
log.info(_("Searching for automatic dependencies for %s...") % self.friendly_name)
# Search for provides.
res = builder.run_script("find-provides", path, filelist)
provides = set(res.splitlines())
# Search for requires.
res = builder.run_script("find-requires", path, filelist)
requires = set(res.splitlines()) - provides
finally:
os.unlink(filelist)
self._dependencies["provides"] = provides
self._dependencies["requires"] = requires
# Filter dependencies.
for key in ("prerequires", "requires", "provides", "conflicts", "obsoletes"):
# Make sure this is a list.
try:
self._dependencies[key] = list(self._dependencies[key])
except KeyError:
self._dependencies[key] = []
# Filter out unwanted elements.
self._dependencies[key] = self.filter_deps(
self._dependencies[key], self.lexer.get_var("filter_%s" % key)
)
def create_datafile(self):
# Create a list of all files that have to be put into the
# package.
files = []
# Download all files that go into the package.
for file in self.pkg.download():
assert os.path.getsize(file), "Don't package empty files"
files.append(("files/%s" % os.path.basename(file), file))
# Add all files in the package directory.
for file in self.pkg.files:
files.append((os.path.relpath(file, self.pkg.path), file))
# Add files in alphabetical order.
files.sort()
# Load progressbar.
message = "%-10s : %s" % (_("Packaging"), self.pkg.friendly_name)
pb = util.make_progress(message, len(files), eta=False)
filename = self.mktemp()
if self.payload_compression == "xz":
datafile = tar.InnerTarFileXz.open(filename, mode="w")
else:
datafile = tar.InnerTarFile.open(filename, mode="w")
i = 0
for arcname, file in files:
if pb:
i += 1
pb.update(i)
datafile.add(file, arcname)
datafile.close()
if pb:
pb.finish()
return filename
def open(self):
# Find all files in the repository dir.
files = self.search_files(self.path)
# Create progress bar.
pb = util.make_progress(_("%s: Reading packages...") % self.name, len(files))
i = 0
# Add all files to the index.
for file in files:
if pb:
i += 1
pb.update(i)
pkg = packages.open(self.pakfire, self, file)
self.index.add_package(pkg)
if pb:
pb.finish()
# Mark repo as open.
self.opened = True
def restart_keepalive(self, wait=None):
log.critical(_("Restarting keepalive process"))
# Send SIGTERM to really end the process.
self.keepalive.terminate()
# Wait for the process to terminate.
if wait:
self.keepalive.join(wait)
# Remove the keepalive process from the process list.
try:
self.__workers.remove(self.keepalive)
except ValueError:
pass
# Create a new process and start it.
self.keepalive = PakfireDaemonKeepalive(self.config)
self.keepalive.start()
# Add the process to the process list.
self.__workers.append(self.keepalive)
def send_builder_info(self):
log.info(_("Sending builder information to hub..."))
data = {
# CPU info
"cpu_model" : system.cpu_model,
"cpu_count" : system.cpu_count,
"cpu_arch" : system.native_arch,
"cpu_bogomips" : system.cpu_bogomips,
# Memory + swap
"mem_total" : system.memory,
"swap_total" : system.swap_total,
# Pakfire + OS
"pakfire_version" : PAKFIRE_VERSION,
"host_key" : self.config.get("signatures", "host_key", None),
"os_name" : system.distro.pretty_name,
# Supported arches
"supported_arches" : ",".join(system.supported_arches),
}
self.transport.post("/builders/info", data=data)
def open(self):
# Find all files in the repository dir.
files = self.search_files(self.path)
# Create progress bar.
pb = util.make_progress(
_("%s: Reading packages...") % self.name, len(files))
i = 0
# Add all files to the index.
for file in files:
if pb:
i += 1
pb.update(i)
pkg = packages.open(self.pakfire, self, file)
self.index.add_package(pkg)
if pb:
pb.finish()
# Mark repo as open.
self.opened = True
def info(self):
ret = []
ret.append("")
ret.append(" PAKFIRE %s" % PAKFIRE_VERSION)
ret.append("")
ret.append(" %-20s: %s" % (_("Hostname"), self.hostname))
ret.append("")
# Hardware information
ret.append(" %s:" % _("Hardware information"))
ret.append(" %-16s: %s" % (_("CPU model"), self.cpu_model))
ret.append(" %-16s: %s" % (_("Memory"), pakfire.util.format_size(self.memory)))
ret.append("")
ret.append(" %-16s: %s" % (_("Native arch"), system.native_arch))
header = _("Supported arches")
for arch in self.config.supported_arches:
ret.append(" %-16s: %s" % (header, arch))
header = ""
ret.append("")
return ret
def create_datafile(self):
includes = []
excludes = []
# List of all patterns, which grows.
patterns = self.pkg.files
# ...
orphan_directories = []
for d in ORPHAN_DIRECTORIES:
if d.startswith("usr/"):
b = os.path.basename(d)
b = os.path.join(self.buildroot, b)
if os.path.islink(b):
continue
d = os.path.join(self.buildroot, d)
if not os.path.islink(d):
orphan_directories.append(d)
for pattern in patterns:
# Check if we are running in include or exclude mode.
if pattern.startswith("!"):
files = excludes
# Strip the ! character.
pattern = pattern[1:]
else:
files = includes
# Expand file to point to chroot.
if pattern.startswith("/"):
pattern = pattern[1:]
pattern = os.path.join(self.buildroot, pattern)
# Recognize the type of the pattern. Patterns could be a glob
# pattern that is expanded here or just a directory which will
# be included recursively.
if "*" in pattern or "?" in pattern or ("[" in pattern and "]" in pattern):
_patterns = glob.glob(pattern)
else:
_patterns = [pattern,]
for pattern in _patterns:
# Try to stat the pattern. If that is not successful, we cannot go on.
try:
os.lstat(pattern)
except OSError:
continue
# Add directories recursively but skip those symlinks
# that point to a directory.
if os.path.isdir(pattern) and not os.path.islink(pattern):
# Add directory itself.
files.append(pattern)
for dir, subdirs, _files in os.walk(pattern):
for subdir in subdirs:
if subdir in orphan_directories:
continue
subdir = os.path.join(dir, subdir)
files.append(subdir)
for file in _files:
file = os.path.join(dir, file)
files.append(file)
# All other files are just added.
else:
files.append(pattern)
files = []
for file in includes:
# Skip if file is already in the file set or
# marked to be excluded from this archive.
if file in excludes or file in files:
continue
# Skip orphan directories.
if file in orphan_directories and not os.listdir(file):
log.debug("Found an orphaned directory: %s" % file)
continue
files.append(file)
while True:
file = os.path.dirname(file)
if file == self.buildroot:
break
if not file in files:
files.append(file)
files.sort()
# Load progressbar.
message = "%-10s : %s" % (_("Packaging"), self.pkg.friendly_name)
pb = util.make_progress(message, len(files), eta=False)
datafile = self.mktemp()
if self.payload_compression == "xz":
t = tar.InnerTarFileXz.open(datafile, mode="w")
else:
t = tar.InnerTarFile.open(datafile, mode="w")
# All files in the tarball are relative to this directory.
basedir = self.buildroot
i = 0
for file in files:
if pb:
i += 1
pb.update(i)
# Never package /.
if os.path.normpath(file) == os.path.normpath(basedir):
continue
# Name of the file in the archive.
arcname = "/%s" % os.path.relpath(file, basedir)
# Add file to tarball.
t.add(file, arcname=arcname, recursive=False)
# Remove all packaged files.
for file in reversed(files):
# It's okay if we cannot remove directories,
# when they are not empty.
if os.path.isdir(file):
try:
os.rmdir(file)
except OSError:
continue
else:
try:
os.unlink(file)
except OSError:
pass
while True:
file = os.path.dirname(file)
if not file.startswith(basedir):
break
try:
os.rmdir(file)
except OSError:
break
# Close the tarfile.
t.close()
# Finish progressbar.
if pb:
pb.finish()
return datafile
def download(self, pkg, text="", logger=None):
"""
Downloads 'filename' from repository and returns the local filename.
"""
if logger is None:
logger = log
filename, hash1 = pkg.filename, pkg.hash1
# Marker, if we need to download the package.
download = True
cache_filename = pkg.cache_filename
# Check if file already exists in cache.
if self.cache.exists(cache_filename):
logger.debug("File exists in cache: %s" % filename)
# If the file does already exist, we check if the hash1 matches.
if hash1 and self.cache.verify(cache_filename, hash1):
# We already got the right file. Skip download.
download = False
else:
# The file in cache has a wrong hash. Remove it and repeat download.
self.cache.remove(cache_filename)
# Get a package grabber and add mirror download capabilities to it.
grabber = downloader.PackageDownloader(
self.pakfire,
text=text + os.path.basename(filename),
)
grabber = self.mirrors.group(grabber)
# Make sure filename is of type string (and not unicode)
filename = str(filename)
while download:
logger.debug("Going to download %s" % filename)
# If we are in offline mode, we cannot download any files.
if self.pakfire.offline and not self.baseurl.startswith("file://"):
raise OfflineModeError, _("Cannot download this file in offline mode: %s") \
% filename
try:
i = grabber.urlopen(filename)
except urlgrabber.grabber.URLGrabError, e:
raise DownloadError, _("Could not download %s: %s") % (filename, e)
# Open input and output files and download the file.
o = self.cache.open(cache_filename, "w")
buf = i.read(BUFFER_SIZE)
while buf:
o.write(buf)
buf = i.read(BUFFER_SIZE)
i.close()
o.close()
# Calc the hash1 of the downloaded file.
calc_hash1 = self.cache.hash1(cache_filename)
if calc_hash1 == hash1:
logger.debug("Successfully downloaded %s (%s)." % (filename, hash1))
break
sums = {
"good" : hash1,
"bad" : calc_hash1,
}
logger.warning(_("The checksum of the downloaded file did not match."))
logger.warning(_("Expected %(good)s but got %(bad)s.") % sums)
logger.warning(_("Trying an other mirror."))
# Remove the bad file.
self.cache.remove(cache_filename)
# Go to the next mirror.
grabber.increment_mirror(grabber)
def download(self, pkg, text="", logger=None):
"""
Downloads 'filename' from repository and returns the local filename.
"""
if logger is None:
logger = log
filename, hash1 = pkg.filename, pkg.hash1
# Marker, if we need to download the package.
download = True
cache_filename = pkg.cache_filename
# Check if file already exists in cache.
if self.cache.exists(cache_filename):
logger.debug("File exists in cache: %s" % filename)
# If the file does already exist, we check if the hash1 matches.
if hash1 and self.cache.verify(cache_filename, hash1):
# We already got the right file. Skip download.
download = False
else:
# The file in cache has a wrong hash. Remove it and repeat download.
self.cache.remove(cache_filename)
# Get a package grabber and add mirror download capabilities to it.
grabber = downloader.PackageDownloader(
self.pakfire,
text=text + os.path.basename(filename),
)
grabber = self.mirrors.group(grabber)
# Make sure filename is of type string (and not unicode)
filename = str(filename)
while download:
logger.debug("Going to download %s" % filename)
# If we are in offline mode, we cannot download any files.
if self.pakfire.offline and not self.baseurl.startswith("file://"):
raise OfflineModeError, _("Cannot download this file in offline mode: %s") \
% filename
try:
i = grabber.urlopen(filename)
except urlgrabber.grabber.URLGrabError, e:
raise DownloadError, _("Could not download %s: %s") % (
filename, e)
# Open input and output files and download the file.
o = self.cache.open(cache_filename, "w")
buf = i.read(BUFFER_SIZE)
while buf:
o.write(buf)
buf = i.read(BUFFER_SIZE)
i.close()
o.close()
# Calc the hash1 of the downloaded file.
calc_hash1 = self.cache.hash1(cache_filename)
if calc_hash1 == hash1:
logger.debug("Successfully downloaded %s (%s)." %
(filename, hash1))
break
sums = {
"good": hash1,
"bad": calc_hash1,
}
logger.warning(
_("The checksum of the downloaded file did not match."))
logger.warning(_("Expected %(good)s but got %(bad)s.") % sums)
logger.warning(_("Trying an other mirror."))
# Remove the bad file.
self.cache.remove(cache_filename)
# Go to the next mirror.
grabber.increment_mirror(grabber)
def save(self, path=None, algo="xz"):
"""
This function saves the database and metadata to path so it can
be exported to a remote repository.
"""
if not path:
path = self.path
# Create filenames
metapath = os.path.join(path, METADATA_DOWNLOAD_PATH)
db_path = os.path.join(metapath, METADATA_DATABASE_FILE)
md_path = os.path.join(metapath, METADATA_DOWNLOAD_FILE)
# Remove all pre-existing metadata.
if os.path.exists(metapath):
util.rm(metapath)
# Create directory for metdadata.
os.makedirs(metapath)
# Save the database to path and get the filename.
self.index.write(db_path)
# Make a reference to the database file that it will get a unique name
# so we won't get into any trouble with caching proxies.
db_hash = util.calc_hash1(db_path)
db_path2 = os.path.join(os.path.dirname(db_path),
"%s-%s" % (db_hash, os.path.basename(db_path)))
# Compress the database.
if algo:
# Open input file and get filesize of input file.
f = open(db_path)
filesize = os.path.getsize(db_path)
# Make a nice progress bar.
p = util.make_progress(_("Compressing database..."), filesize)
# Create compressing file handler.
c = compress.compressobj(db_path2)
try:
size = 0
while True:
buf = f.read(BUFFER_SIZE)
if not buf:
break
if p:
size += len(buf)
p.update(size)
c.write(buf)
except:
# XXX catch compression errors
raise
finally:
f.close()
c.close()
p.finish()
# Remove old database.
os.unlink(db_path)
else:
shutil.move(db_path, db_path2)
# Create a new metadata object and add out information to it.
md = metadata.Metadata(self.pakfire)
# Save name of the hashed database to the metadata.
md.database = os.path.basename(db_path2)
md.database_hash1 = db_hash
md.database_compression = algo
# Save metdata to repository.
md.save(md_path)
def extract(self, message=None, prefix=None):
log.debug("Extracting package %s" % self.friendly_name)
if prefix is None:
prefix = ""
# Open package data for read.
payload_archive = self.open_payload_archive()
# Load progressbar.
pb = None
if message:
message = "%-10s : %s" % (message, self.friendly_name)
pb = util.make_progress(message, len(self.filelist), eta=False)
# Collect messages with errors and warnings, that are passed to
# the user.
messages = []
name2file = {}
for file in self.filelist:
if file.is_dir() and file.name.endswith("/"):
name = file.name[:-1]
else:
name = file.name
name2file[name] = file
i = 0
while True:
member = payload_archive.next()
if not member:
break
# Check if file is also known in metadata.
name = member.name
if not name.startswith("/"):
name = "/%s" % name
try:
file = name2file[name]
except KeyError:
log.warning(_("File in archive is missing in file metadata: %s. Skipping.") % name)
continue
# Update progress.
if pb:
i += 1
pb.update(i)
target = os.path.join(prefix, member.name)
# Check if a configuration file is already present. We don't want to
# overwrite that.
if file.is_config():
config_save = "%s%s" % (target, CONFIG_FILE_SUFFIX_SAVE)
config_new = "%s%s" % (target, CONFIG_FILE_SUFFIX_NEW)
if os.path.exists(config_save) and not os.path.exists(target):
# Extract new configuration file, save it as CONFIG_FILE_SUFFIX_NEW,
# and reuse _SAVE.
payload_archive.extract(member, path=prefix)
shutil.move(target, config_new)
shutil.move(config_save, target)
continue
elif os.path.exists(target):
# If the files are identical, we skip the extraction of a
# new configuration file. We also do that when the new configuration file
# is a dummy file.
if file.size == 0:
continue
# Calc hash of the current configuration file.
config_hash1 = hashlib.new("sha512")
f = open(target)
while True:
buf = f.read(BUFFER_SIZE)
if not buf:
break
config_hash1.update(buf)
f.close()
if file.hash1 == config_hash1.hexdigest():
continue
# Backup old configuration file and extract new one.
shutil.move(target, config_save)
payload_archive.extract(member, path=prefix)
# Save new configuration file as CONFIG_FILE_SUFFIX_NEW and
# restore old configuration file.
shutil.move(target, config_new)
shutil.move(config_save, target)
if prefix:
config_new = os.path.relpath(config_new, prefix)
messages.append(_("Config file created as %s") % config_new)
continue
# Don't overwrite target files if they already exist.
if file.is_datafile() and os.path.exists(target):
log.debug(_("Don't overwrite already existing datafile '/%s'") % member.name)
continue
# If the member is a directory and if it already exists, we
# don't need to create it again.
if os.path.exists(target):
if member.isdir():
continue
else:
# Remove file if it has been existant
try:
os.unlink(target)
except OSError:
messages.append(_("Could not remove file: /%s") % member.name)
#if self.pakfire.config.get("debug"):
# msg = "Creating file (%s:%03d:%03d) " % \
# (tarfile.filemode(member.mode), member.uid, member.gid)
# if member.issym():
# msg += "/%s -> %s" % (member.name, member.linkname)
# elif member.islnk():
# msg += "/%s link to /%s" % (member.name, member.linkname)
# else:
# msg += "/%s" % member.name
# log.debug(msg)
payload_archive.extract(member, path=prefix)
# Close all open files.
payload_archive.close()
if pb:
pb.finish()
# Print messages.
for msg in messages:
log.warning(msg)
def _remove_files(self, files, message, prefix):
if prefix in ("/", None):
prefix = ""
# Load progressbar.
pb = None
if message:
message = "%-10s : %s" % (message, self.friendly_name)
pb = util.make_progress(message, len(files), eta=False)
# Sort files by the length of their name to remove all files in
# a directory first and then check, if there are any files left.
files.sort(cmp=lambda x,y: cmp(len(x.name), len(y.name)), reverse=True)
# Messages to the user.
messages = []
i = 0
for _file in files:
# Update progress.
if pb:
i += 1
pb.update(i)
log.debug("Removing file: %s" % _file)
if prefix:
file = os.path.join(prefix, _file.name[1:])
assert file.startswith("%s/" % prefix)
else:
file = _file.name
# Rename configuration files.
if _file.is_config():
# Skip already removed config files.
try:
os.lstat(file)
except OSError:
continue
file_save = "%s%s" % (file, CONFIG_FILE_SUFFIX_SAVE)
try:
shutil.move(file, file_save)
except shutil.Error, e:
print e
if prefix:
file_save = os.path.relpath(file_save, prefix)
messages.append(_("Config file saved as %s.") % file_save)
continue
# Preserve datafiles.
if _file.is_datafile():
log.debug(_("Preserving datafile '/%s'") % _file)
continue
# Handle regular files and symlinks.
if os.path.isfile(file) or os.path.islink(file):
log.debug("Removing %s..." % _file)
try:
os.remove(file)
except OSError:
log.error("Cannot remove file: %s. Remove manually." % _file)
# Handle directories.
# Skip removal if the directory is a mountpoint.
elif os.path.isdir(file) and not os.path.ismount(file):
# Try to remove the directory. If it is not empty, OSError is raised,
# but we are okay with that.
try:
os.rmdir(file)
except OSError:
pass
# Handle files that have already been removed
# by somebody else.
elif not os.path.exists(file):
pass
# Log all unhandled types.
else:
log.warning("Cannot remove file: %s. Filetype is unhandled." % file)
def migrate(self):
# If we have already the latest version, there is nothing to do.
if self.format == DATABASE_FORMAT:
return
# Check if database version is supported.
if self.format > DATABASE_FORMAT:
raise DatabaseError, _(
"Cannot use database with version greater than %s."
) % DATABASE_FORMAT
log.info(_("Migrating database from format %(old)s to %(new)s.") % \
{ "old" : self.format, "new" : DATABASE_FORMAT })
# Get a database cursor.
c = self.cursor()
# 1) The vendor column was added.
if self.format < 1:
c.execute("ALTER TABLE packages ADD COLUMN vendor TEXT AFTER uuid")
if self.format < 2:
c.execute("ALTER TABLE files ADD COLUMN `config` INTEGER")
c.execute("ALTER TABLE files ADD COLUMN `mode` INTEGER")
c.execute("ALTER TABLE files ADD COLUMN `user` TEXT")
c.execute("ALTER TABLE files ADD COLUMN `group` TEXT")
c.execute("ALTER TABLE files ADD COLUMN `mtime` INTEGER")
if self.format < 3:
c.execute("ALTER TABLE files ADD COLUMN `capabilities` TEXT")
if self.format < 4:
c.execute(
"ALTER TABLE packages ADD COLUMN recommends TEXT AFTER obsoletes"
)
c.execute(
"ALTER TABLE packages ADD COLUMN suggests TEXT AFTER recommends"
)
if self.format < 5:
c.execute(
"ALTER TABLE files ADD COLUMN datafile INTEGER AFTER config")
if self.format < 6:
c.execute(
"ALTER TABLE packages ADD COLUMN inst_size INTEGER AFTER size")
if self.format < 7:
c.executescript("""
CREATE TABLE dependencies(pkg INTEGER, type TEXT, dependency TEXT);
CREATE INDEX dependencies_pkg_index ON dependencies(pkg);
""")
c.execute(
"SELECT id, provides, requires, conflicts, obsoletes, recommends, suggests FROM packages"
)
pkgs = c.fetchall()
for pkg in pkgs:
(pkg_id, provides, requires, conflicts, obsoletes, recommends,
suggests) = pkg
dependencies = (
("provides", provides),
("requires", requires),
("conflicts", conflicts),
("obsoletes", obsoletes),
("recommends", recommends),
("suggests", suggests),
)
for type, deps in dependencies:
c.executemany(
"INSERT INTO dependencies(pkg, type, dependency) VALUES(?, ?, ?)",
((pkg_id, type, d) for d in deps.splitlines()))
c.executescript("""
CREATE TABLE packages_(
id INTEGER PRIMARY KEY,
name TEXT,
epoch INTEGER,
version TEXT,
release TEXT,
arch TEXT,
groups TEXT,
filename TEXT,
size INTEGER,
inst_size INTEGER,
hash1 TEXT,
license TEXT,
summary TEXT,
description TEXT,
uuid TEXT,
vendor TEXT,
build_id TEXT,
build_host TEXT,
build_date TEXT,
build_time INTEGER,
installed INT,
reason TEXT,
repository TEXT
);
INSERT INTO packages_ SELECT id, name, epoch, version, release, arch, groups, filename,
size, inst_size, hash1, license, summary, description, uuid, vendor, build_id,
build_host, build_date, build_time, installed, reason, repository FROM packages;
DROP TABLE packages;
ALTER TABLE packages_ RENAME TO packages;
DROP TABLE triggers;
CREATE INDEX files_pkg_index ON files(pkg);
CREATE INDEX scriptlets_pkg_index ON scriptlets(pkg);
CREATE INDEX packages_name_index ON packages(name);
""")
# In the end, we can easily update the version of the database.
c.execute("UPDATE settings SET val = ? WHERE key = 'version'",
(DATABASE_FORMAT, ))
self.__format = DATABASE_FORMAT
self.commit()
c.close()
def execute(self):
# Save start time.
self.time_start = time.time()
if self.logger:
self.logger.debug(
_("Executing command: %s in %s") %
(self.command, self.chroot_path or "/"))
child = None
try:
# Create new child process
child = self.create_subprocess()
# Record the output.
self.tee_log(child)
except:
# In case there has been an error, kill children if they aren't done
if child and child.returncode is None:
os.killpg(child.pid, 9)
try:
if child:
os.waitpid(child.pid, 0)
except:
pass
# Raise original exception.
raise
finally:
# Save end time.
self.time_end = time.time()
# wait until child is done, kill it if it passes timeout
nice_exit = True
while child.poll() is None:
if self.timeout_has_been_exceeded():
nice_exit = False
os.killpg(child.pid, 15)
if self.timeout_has_been_exceeded(3):
nice_exit = False
os.killpg(child.pid, 9)
if not nice_exit:
raise commandTimeoutExpired, (
_("Command exceeded timeout (%(timeout)d): %(command)s") %
(self.timeout, self.command))
# Save exitcode.
self.exitcode = child.returncode
if self.logger:
self.logger.debug(_("Child returncode was: %s") % self.exitcode)
if self.exitcode and self.log_errors:
raise ShellEnvironmentError, (_("Command failed: %s") %
self.command, self.exitcode)
return self.exitcode
def dump(self, short=False, long=False, filelist=False):
if short:
return "%s.%s : %s" % (self.name, self.arch, self.summary)
items = [
(_("Name"), self.name),
]
# Show supported arches if available.
if hasattr(self, "supported_arches") and not self.supported_arches == "all":
arch = "%s (%s)" % (self.arch, self.supported_arches)
else:
arch = self.arch
items.append((_("Arch"), arch))
items += [
(_("Version"), self.version),
(_("Release"), self.release),
]
if self.size:
items.append((_("Size"), util.format_size(self.size)))
if self.inst_size:
items.append(
(_("Installed size"),
util.format_size(self.inst_size))
)
# filter dummy repository
if not self.repo == self.pakfire.repos.dummy:
items.append((_("Repo"), self.repo.name))
items += [
(_("Summary"), self.summary),
(_("Groups"), " ".join(self.groups)),
(_("URL"), self.url),
(_("License"), self.license),
]
caption = _("Description")
for line in util.text_wrap(self.description):
items.append((caption, line))
caption = ""
if long:
if self.maintainer:
items.append((_("Maintainer"), self.maintainer))
items.append((_("Vendor"), self.vendor))
items.append((_("UUID"), self.uuid))
items.append((_("Build ID"), self.build_id))
items.append((_("Build date"), self.build_date))
items.append((_("Build host"), self.build_host))
caption = _("Signatures")
for sig in self.signatures:
items.append((caption, sig))
caption = ""
caption = _("Provides")
for prov in sorted(self.provides):
items.append((caption, prov))
caption = ""
caption = _("Pre-requires")
for req in sorted(self.prerequires):
items.append((caption, req))
caption = ""
caption = _("Requires")
for req in sorted(self.requires):
items.append((caption, req))
caption = ""
caption = _("Conflicts")
for req in sorted(self.conflicts):
items.append((caption, req))
caption = ""
caption = _("Obsoletes")
for req in sorted(self.obsoletes):
items.append((caption, req))
caption = ""
caption = _("Recommends")
for req in sorted(self.recommends):
items.append((caption, req))
caption = ""
caption = _("Suggests")
for req in sorted(self.suggests):
items.append((caption, req))
caption = ""
# Append filelist if requested.
if filelist:
for file in self.filelist:
items.append((_("File"), file))
format = "%%-%ds : %%s" % (max([len(k) for k, v in items]))
s = []
for caption, value in items:
s.append(format % (caption, value))
s.append("") # New line at the end
# XXX why do we need to decode this?
return "\n".join([str.decode("utf-8") for str in s])
def save(self, path=None, algo="xz"):
"""
This function saves the database and metadata to path so it can
be exported to a remote repository.
"""
if not path:
path = self.path
# Create filenames
metapath = os.path.join(path, METADATA_DOWNLOAD_PATH)
db_path = os.path.join(metapath, METADATA_DATABASE_FILE)
md_path = os.path.join(metapath, METADATA_DOWNLOAD_FILE)
# Remove all pre-existing metadata.
if os.path.exists(metapath):
util.rm(metapath)
# Create directory for metdadata.
os.makedirs(metapath)
# Save the database to path and get the filename.
self.index.write(db_path)
# Make a reference to the database file that it will get a unique name
# so we won't get into any trouble with caching proxies.
db_hash = util.calc_hash1(db_path)
db_path2 = os.path.join(os.path.dirname(db_path),
"%s-%s" % (db_hash, os.path.basename(db_path)))
# Compress the database.
if algo:
# Open input file and get filesize of input file.
f = open(db_path)
filesize = os.path.getsize(db_path)
# Make a nice progress bar.
p = util.make_progress(_("Compressing database..."), filesize)
# Create compressing file handler.
c = compress.compressobj(db_path2)
try:
size = 0
while True:
buf = f.read(BUFFER_SIZE)
if not buf:
break
if p:
size += len(buf)
p.update(size)
c.write(buf)
except:
# XXX catch compression errors
raise
finally:
f.close()
c.close()
p.finish()
# Remove old database.
os.unlink(db_path)
else:
shutil.move(db_path, db_path2)
# Create a new metadata object and add out information to it.
md = metadata.Metadata(self.pakfire)
# Save name of the hashed database to the metadata.
md.database = os.path.basename(db_path2)
md.database_hash1 = db_hash
md.database_compression = algo
# Save metdata to repository.
md.save(md_path)
