def get_coin_features(self):
"set the global variable SWIG_COND_SYMBOLS needed for conditional " + \
"wrapping"
if sys.platform == "win32": return
print blue("Checking for Coin features..."),
if not os.system("coin-config --have-feature 3ds_import"):
self.SWIG_COND_SYMBOLS.append("-DHAVE_FEATURE_3DS_IMPORT")
print green("3ds import "),
if not os.system("coin-config --have-feature vrml97"):
self.SWIG_COND_SYMBOLS.append("-DHAVE_FEATURE_VRML97")
print green("vrml97 "),
if not os.system("coin-config --have-feature sound"):
self.SWIG_COND_SYMBOLS.append("-DHAVE_FEATURE_SOUND")
print green("sound "),
if not os.system("coin-config --have-feature superglu"):
self.SWIG_COND_SYMBOLS.append("-DHAVE_FEATURE_SUPERGLUE")
print green("superglu "),
if not os.system("coin-config --have-feature threads"):
self.SWIG_COND_SYMBOLS.append("-DHAVE_FEATURE_THREADS")
print green("threads "),
if not os.system("coin-config --have-feature threadsafe"):
self.SWIG_COND_SYMBOLS.append("-DHAVE_FEATURE_THREADSAFE")
print green("threadsafe "),
print
def check_cmd_exists(self, cmd):
"return the path of the specified command if it exists"
print blue("Checking for %s..." % cmd),
for path in os.environ['PATH'].split(os.path.pathsep):
if os.path.exists(os.path.join(path, cmd)):
print blue("'%s'" % os.path.join(path, cmd))
return 1
print red("not found.")
return 0
def check_coin_version(self):
"check the Coin version"
if sys.platform == "win32": return
if not self.check_cmd_exists("coin-config"):
sys.exit(1)
print blue("Coin version..."),
version = self.do_os_popen("coin-config --version")
print blue("%s" % version)
if not version.startswith('3'):
print yellow("** Warning: Pivy has only been tested with Coin "
"versions Coin-dev 3.")
def swig_generate(self):
"build all available modules"
quote = lambda s : '"' + s + '"'
for module in self.MODULES:
module_name = self.MODULES[module][0]
config_cmd = self.MODULES[module][1]
module_pkg_name = self.MODULES[module][2]
mod_out_prefix = module_pkg_name.replace('.', os.sep) + module
if sys.platform == "win32":
INCLUDE_DIR = os.path.join(os.getenv("COINDIR"), "include")
CPP_FLAGS = "-I" + quote(INCLUDE_DIR) + " " + \
"-I" + quote(os.path.join(os.getenv("COINDIR"), "include", "Inventor", "annex")) + \
" /DCOIN_DLL /wd4244 /wd4049"
# aquire highest non-debug Coin library version
LDFLAGS_LIBS = quote(max(glob.glob(os.path.join(os.getenv("COINDIR"), "lib", "coin?.lib")))) + " "
if module == "sowin":
CPP_FLAGS += " /DSOWIN_DLL"
LDFLAGS_LIBS += quote(os.path.join(os.getenv("COINDIR"), "lib", "sowin1.lib"))
elif module == "soqt":
CPP_FLAGS += " -I" + '"' + os.getenv("QTDIR") + "\\include\" /DSOQT_DLL"
CPP_FLAGS += " -I" + '"' + os.getenv("QTDIR") + "\\include\Qt\""
LDFLAGS_LIBS += os.path.join(os.getenv("COINDIR"), "lib", "soqt1.lib") + " "
else:
INCLUDE_DIR = self.do_os_popen("coin-config --includedir")
if module_name != 'coin':
mod_include_dir = self.do_os_popen("%s --includedir" % config_cmd)
if mod_include_dir != INCLUDE_DIR:
INCLUDE_DIR += '\" -I\"%s' % mod_include_dir
CPP_FLAGS = self.do_os_popen("%s --cppflags" % config_cmd)
LDFLAGS_LIBS = self.do_os_popen("%s --ldflags --libs" % config_cmd)
if not os.path.isfile(mod_out_prefix + "_wrap.cpp"):
print red("\n=== Generating %s_wrap.cpp for %s ===\n" %
(mod_out_prefix, module))
print blue(self.SWIG + " " + self.SWIG_SUPPRESS_WARNINGS + " " + self.SWIG_PARAMS %
(INCLUDE_DIR,
self.CXX_INCS,
mod_out_prefix, module))
if os.system(self.SWIG + " " + self.SWIG_SUPPRESS_WARNINGS + " " + self.SWIG_PARAMS %
(INCLUDE_DIR,
self.CXX_INCS,
mod_out_prefix, module)):
print red("SWIG did not generate wrappers successfully! ** Aborting **")
sys.exit(1)
else:
print red("=== %s_wrap.cpp for %s already exists! ===" % (mod_out_prefix, module_pkg_name + module))
self.ext_modules.append(Extension(module_name, [mod_out_prefix + "_wrap.cpp"],
extra_compile_args=(self.CXX_INCS + CPP_FLAGS).split(),
extra_link_args=(self.CXX_LIBS + LDFLAGS_LIBS).split()))
self.py_modules.append(module_pkg_name + module)
def check_simvoleon_version(self):
"return if SIMVoleon is available and check the version"
if sys.platform == "win32" or not self.check_cmd_exists("simvoleon-config"):
self.MODULES.pop('simvoleon', None)
return False
print blue("SIMVoleon version..."),
version = self.do_os_popen("simvoleon-config --version")
print blue("%s" % version)
if not version.startswith('2.0'):
print yellow("** Warning: Pivy has only been tested with SIMVoleon "
"versions 2.0.x.")
return True
def check_swig_version(self, swig):
"check for the swig version"
global SWIG_VERSION
if not self.check_cmd_exists(swig):
sys.exit(1)
print blue("Checking for SWIG version..."),
p = subprocess.Popen("%s -version" % swig,
shell=True, stdout=subprocess.PIPE)
version = p.stdout.readlines()[1].strip().split(" ")[2]
p.stdout.close()
print blue("%s" % version)
SWIG_VERSION = version
if not version in self.SUPPORTED_SWIG_VERSIONS:
print yellow("Warning: Pivy has only been tested with the following " + \
"SWIG versions: %s." % " ".join(self.SUPPORTED_SWIG_VERSIONS))
def convert_myoldbest(self, pkg, pkg_info):
"""converts and colorizes a version list to a string
@param pkg: _emerge.Package.Package instance
@param pkg_info: dictionary
@rtype string.
"""
myoldbest = pkg_info.oldbest_list
# Convert myoldbest from a list to a string.
myoldbest_str = ""
if myoldbest:
versions = []
for pos, old_pkg in enumerate(myoldbest):
key = old_pkg.version
if key[-3:] == "-r0":
key = key[:-3]
if self.conf.verbosity == 3:
if pkg_info.attr_display.new_slot:
key += _slot_separator + old_pkg.slot
if old_pkg.slot != old_pkg.sub_slot:
key += "/" + old_pkg.sub_slot
elif any(x.slot + "/" + x.sub_slot != "0/0" for x in myoldbest + [pkg]):
key += _slot_separator + old_pkg.slot
if old_pkg.slot != old_pkg.sub_slot or \
old_pkg.slot == pkg.slot and old_pkg.sub_slot != pkg.sub_slot:
key += "/" + old_pkg.sub_slot
if not self.quiet_repo_display and (self.verbose_main_repo_display or
self.portdb.repositories.mainRepo() is None or
any(x.repo != self.portdb.repositories.mainRepo().name for x in myoldbest + [pkg])):
key += _repo_separator + old_pkg.repo
versions.append(key)
myoldbest_str = blue("["+", ".join(versions)+"]")
return myoldbest_str
def _insert_slot(self, pkg, pkg_info, myinslotlist):
"""Adds slot info to the message
@returns addl: formatted slot info
@returns myoldbest: installed version list
Modifies self.counters.downgrades, self.counters.upgrades,
self.counters.binary
"""
addl = " " + pkg_info.fetch_symbol
if not cpvequal(pkg.cpv,
best([pkg.cpv] + [x.cpv for x in myinslotlist])):
# Downgrade in slot
addl += turquoise("U")+blue("D")
if pkg_info.ordered:
self.counters.downgrades += 1
if pkg.type_name == "binary":
self.counters.binary += 1
else:
# Update in slot
addl += turquoise("U") + " "
if pkg_info.ordered:
self.counters.upgrades += 1
if pkg.type_name == "binary":
self.counters.binary += 1
return addl
def _set_no_columns(self, pkg, pkg_info):
"""prints pkg info without column indentation.
@param pkg: _emerge.Package.Package instance
@param pkg_info: dictionary
@rtype the updated addl
"""
pkg_str = pkg.cpv
if self.conf.verbosity == 3:
pkg_str = self._append_slot(pkg_str, pkg, pkg_info)
pkg_str = self._append_repository(pkg_str, pkg, pkg_info)
if not pkg_info.merge:
addl = self.empty_space_in_brackets()
myprint = "[%s%s] %s%s %s" % \
(self.pkgprint(pkg_info.operation.ljust(13),
pkg_info), addl,
self.indent, self.pkgprint(pkg_str, pkg_info),
pkg_info.oldbest)
else:
myprint = "[%s %s%s] %s%s %s" % \
(self.pkgprint(pkg.type_name, pkg_info),
pkg_info.attr_display, \
bold(blue("L")) if self._has_local_patch(pkg) else " ", \
self.indent, \
self.pkgprint(pkg_str, pkg_info), pkg_info.oldbest)
#if self.localpatch_enabled:
#self. += bold(blue("L")) if self._has_local_patch(pkg) else " "
return myprint
def copy_and_swigify_headers(self, includedir, dirname, files):
"""Copy the header files to the local include directories. Add an
#include line at the beginning for the SWIG interface files..."""
for file in files:
if not os.path.isfile(os.path.join(dirname, file)):
continue
if file[-2:] == ".i":
file_i = os.path.join(dirname, file)
file_h = os.path.join(dirname, file)[:-2] + ".h"
if (not os.path.exists(file_h) and
os.path.exists(os.path.join(includedir, file_h))):
shutil.copyfile(os.path.join(includedir, file_h), file_h)
sys.stdout.write(' ' + turquoise(file_h))
fd = open(file_h, 'r+')
contents = fd.readlines()
ins_line_nr = -1
for line in contents:
ins_line_nr += 1
if line.find("#include ") != -1:
break
if ins_line_nr != -1:
contents.insert(ins_line_nr, self.pivy_header_include % (file_i))
fd.seek(0)
fd.writelines(contents)
else:
print blue("[") + red("failed") + blue("]")
sys.exit(1)
fd.close
# fixes for SWIG 1.3.21 and upwards
# (mostly workarounding swig's preprocessor "function like macros"
# preprocessor bug when no parameters are provided which then results
# in no constructors being created in the wrapper)
elif file[-4:] == ".fix":
sys.stdout.write(' ' + red(os.path.join(dirname, file)[:-4]))
shutil.copyfile(os.path.join(dirname, file),
os.path.join(dirname, file)[:-4])
# had to introduce this because windows is a piece of crap
elif sys.platform == "win32" and file[-6:] == ".win32":
sys.stdout.write(' ' + red(os.path.join(dirname, file)[:-6]))
shutil.copyfile(os.path.join(dirname, file),
os.path.join(dirname, file)[:-6])
def keyword(string, stable=True, hard_masked=False): """Returns a keyword string.""" if stable: return output.green(string) if hard_masked: return output.red(string) # keyword masked: return output.blue(string)
def __str__(self): s = self.name if self.enabled: s = red(s) else: s = '-' + s s = blue(s) if self.forced: s = '(%s)' % s return s
def pivy_configure(self):
"configure Pivy"
print turquoise(self.PIVY_SNAKES)
print blue("Platform...%s" % sys.platform)
self.check_python_version()
self.check_swig_version(self.SWIG)
self.check_coin_version()
self.get_coin_features()
if self.SOGUI: self.check_gui_bindings()
if 'simvoleon' in self.MODULES and self.check_simvoleon_version():
if sys.platform == "win32":
INCLUDE_DIR = os.getenv("SIMVOLEONDIR") + "\\include"
else:
INCLUDE_DIR = self.do_os_popen("simvoleon-config --includedir")
sys.stdout.write(blue("Preparing") + green(" VolumeViz ") + blue("headers:"))
os.path.walk("VolumeViz", self.copy_and_swigify_headers,
INCLUDE_DIR)
print green(".")
if sys.platform == "win32":
INCLUDE_DIR = os.path.join(os.getenv("COINDIR"), "include")
else:
INCLUDE_DIR = self.do_os_popen("coin-config --includedir")
sys.stdout.write(blue("Preparing") + green(" Inventor ") + blue("headers:"))
os.path.walk("Inventor", self.copy_and_swigify_headers,
INCLUDE_DIR)
print green(".")
def run(self):
"the entry point for the distutils clean class"
sys.stdout.write(blue("Cleaning headers:"))
os.path.walk("Inventor", self.remove_headers, None)
os.path.walk("VolumeViz", self.remove_headers, None)
# remove the SWIG generated wrappers
for wrapper_file in self.REMOVE_FILES:
if os.path.isfile(wrapper_file):
sys.stdout.write(' ' + turquoise(wrapper_file))
os.remove(wrapper_file)
print green(".")
clean.run(self)
def masking(mask):
"""Returns a 'masked by' string."""
if 'package.mask' in mask or 'profile' in mask:
# use porthole wrap style to help clarify meaning
return output.red("M["+mask[0]+"]")
if mask is not []:
for status in mask:
if 'keyword' in status:
# keyword masked | " [missing keyword] " <=looks better
return output.blue("["+status+"]")
if status in archlist:
return output.green(status)
if 'unknown' in status:
return output.yellow(status)
return output.red(status)
return ''
def collect_binaries_from_dir(dirs, mask, logger):
''' Collects all binaries from specified list of directories.
mask is list of pathes, that are ommited in scanning,
can be eighter single file or entire directory
Returns list of binaries
'''
# contains list of directories found
# allows us to reduce number of fnc calls
found_directories = []
found_files = []
for _dir in dirs:
if _dir in mask:
continue
try:
for listing in os.listdir(_dir):
listing = os.path.join(_dir, listing)
if listing in mask:
continue
if os.path.isdir(listing):
if os.path.islink(listing):
#we do not want scan symlink-directories
pass
else:
found_directories.append(listing)
elif os.path.isfile(listing):
# we're looking for binaries
# and with binaries we do not need links
# thus we can optimize a bit
if not os.path.islink(listing):
prv = os.stat(listing)[stat.ST_MODE]
if prv & stat.S_IXUSR == stat.S_IXUSR or \
prv & stat.S_IXGRP == stat.S_IXGRP or \
prv & stat.S_IXOTH == stat.S_IXOTH:
found_files.append(listing)
except Exception as ex:
logger.debug(
yellow('Exception during binaries collecting: '+
blue('%s') %str(ex)))
if found_directories:
found_files += collect_binaries_from_dir(found_directories, mask, logger)
return found_files
def __str__(self):
output = []
if self.interactive:
output.append(colorize("WARN", "I"))
else:
output.append(" ")
if self.new or self.force_reinstall:
if self.force_reinstall:
output.append(red("r"))
else:
output.append(green("N"))
else:
output.append(" ")
if self.new_slot or self.replace:
if self.replace:
output.append(yellow("R"))
else:
output.append(green("S"))
else:
output.append(" ")
if self.fetch_restrict or self.fetch_restrict_satisfied:
if self.fetch_restrict_satisfied:
output.append(green("f"))
else:
output.append(red("F"))
else:
output.append(" ")
if self.new_version:
output.append(turquoise("U"))
else:
output.append(" ")
if self.downgrade:
output.append(blue("D"))
else:
output.append(" ")
if self.mask is not None:
output.append(self.mask)
return "".join(output)
def convert_myoldbest(myoldbest):
"""converts and colorizes a version list to a string
@param myoldbest: list
@rtype string.
"""
# Convert myoldbest from a list to a string.
myoldbest_str = ""
if myoldbest:
versions = []
for pos, pkg in enumerate(myoldbest):
key = catpkgsplit(pkg.cpv)[2] + \
"-" + catpkgsplit(pkg.cpv)[3]
if key[-3:] == "-r0":
key = key[:-3]
versions.append(key)
myoldbest_str = blue("["+", ".join(versions)+"]")
return myoldbest_str
def do_normal(pkg, verbose):
data = []
if not pkg[4]:
installed = "[ Not Installed ]"
else:
installed = pkg[4]
if pkg[2]:
masked = red(" [ Masked ]")
else:
masked = ""
data.append("%s %s%s\n %s %s\n %s %s" % \
(green("*"), bold(pkg[1]), masked,
darkgreen("Latest version available:"), pkg[3],
darkgreen("Latest version installed:"), installed))
if verbose:
mpv = best(portdb.xmatch("match-all", pkg[1]))
iuse_split, final_use = get_flags(mpv, final_setting=True)
iuse = ""
use_list = []
for ebuild_iuse in iuse_split:
use = ebuild_iuse.lstrip('+-')
if use in final_use:
use_list.append(red("+" + use) + " ")
else:
use_list.append(blue("-" + use) + " ")
use_list.sort()
iuse = ' '.join(use_list)
if iuse == "":
iuse = "-"
data.append(" %s %s\n %s %s" % \
(darkgreen("Unstable version:"), pkg_version(mpv),
darkgreen("Use Flags (stable):"), iuse))
data.append(" %s %s\n %s %s\n %s %s\n %s %s\n" % \
(darkgreen("Size of downloaded files:"), pkg[5],
darkgreen("Homepage:"), pkg[6],
darkgreen("Description:"), pkg[7],
darkgreen("License:"), pkg[8]))
return data
def convert_myoldbest(self, pkg, myoldbest):
"""converts and colorizes a version list to a string
@param pkg: _emerge.Package.Package instance
@param myoldbest: list
@rtype string.
"""
# Convert myoldbest from a list to a string.
myoldbest_str = ""
if myoldbest:
versions = []
for pos, old_pkg in enumerate(myoldbest):
key = catpkgsplit(old_pkg.cpv)[2] + "-" + catpkgsplit(old_pkg.cpv)[3]
if key[-3:] == "-r0":
key = key[:-3]
if self.conf.verbosity == 3 and not self.quiet_repo_display and (self.verbose_main_repo_display or
any(x.repo != self.portdb.repositories.mainRepo().name for x in myoldbest + [pkg])):
key += _repo_separator + old_pkg.repo
versions.append(key)
myoldbest_str = blue("["+", ".join(versions)+"]")
return myoldbest_str
def check_gui_bindings(self):
"check for availability of SoGui bindings and removes the not available ones"
if sys.platform == "win32":
print "Coin and SoWin are built by default on Windows..."
self.MODULES.pop('soxt', None)
self.MODULES.pop('sogtk', None)
print blue("Checking for QTDIR environment variable..."),
if os.getenv("QTDIR"):
print blue(os.getenv("QTDIR"))
else:
self.MODULES.pop('soqt', None)
print red("not set. (SoQt bindings won't be built)")
else:
for gui in self.SOGUI:
if not self.MODULES.has_key(gui):
continue
gui_config_cmd = self.MODULES[gui][1]
if not self.check_cmd_exists(gui_config_cmd):
self.MODULES.pop(gui, None)
else:
print blue("Checking for %s version..." % gui),
version = self.do_os_popen("%s --version" % gui_config_cmd)
print blue("%s" % version)
def update(self):
'''Internal update function which performs the transfer'''
opts = self.options.get('emerge_config').opts
self.usersync_uid = self.options.get('usersync_uid', None)
enter_invalid = '--ask-enter-invalid' in opts
out = portage.output.EOutput()
syncuri = self.repo.sync_uri
if self.repo.module_specific_options.get('sync-rsync-vcs-ignore',
'false').lower() == 'true':
vcs_dirs = ()
else:
vcs_dirs = frozenset(VCS_DIRS)
vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
for vcs_dir in vcs_dirs:
writemsg_level(("!!! %s appears to be under revision " + \
"control (contains %s).\n!!! Aborting rsync sync "
"(override with \"sync-rsync-vcs-ignore = true\" in repos.conf).\n") % \
(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
return (1, False)
self.timeout = 180
rsync_opts = []
if self.settings["PORTAGE_RSYNC_OPTS"] == "":
rsync_opts = self._set_rsync_defaults()
else:
rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
self.extra_rsync_opts = list()
if self.repo.module_specific_options.get('sync-rsync-extra-opts'):
self.extra_rsync_opts.extend(
portage.util.shlex_split(
self.repo.module_specific_options['sync-rsync-extra-opts'])
)
# Process GLEP74 verification options.
# Default verification to 'no'; it's enabled for ::gentoo
# via default repos.conf though.
self.verify_metamanifest = (self.repo.module_specific_options.get(
'sync-rsync-verify-metamanifest', 'no') in ('yes', 'true'))
# Support overriding job count.
self.verify_jobs = self.repo.module_specific_options.get(
'sync-rsync-verify-jobs', None)
if self.verify_jobs is not None:
try:
self.verify_jobs = int(self.verify_jobs)
if self.verify_jobs <= 0:
raise ValueError(self.verify_jobs)
except ValueError:
writemsg_level(
"!!! sync-rsync-verify-jobs not a positive integer: %s\n" %
(self.verify_jobs, ),
level=logging.WARNING,
noiselevel=-1)
self.verify_jobs = None
# Support overriding max age.
self.max_age = self.repo.module_specific_options.get(
'sync-rsync-verify-max-age', '')
if self.max_age:
try:
self.max_age = int(self.max_age)
if self.max_age < 0:
raise ValueError(self.max_age)
except ValueError:
writemsg_level(
"!!! sync-rsync-max-age must be a non-negative integer: %s\n"
% (self.max_age, ),
level=logging.WARNING,
noiselevel=-1)
self.max_age = 0
else:
self.max_age = 0
openpgp_env = None
if self.verify_metamanifest and gemato is not None:
# Use isolated environment if key is specified,
# system environment otherwise
if self.repo.sync_openpgp_key_path is not None:
openpgp_env = gemato.openpgp.OpenPGPEnvironment()
else:
openpgp_env = gemato.openpgp.OpenPGPSystemEnvironment()
try:
# Load and update the keyring early. If it fails, then verification
# will not be performed and the user will have to fix it and try again,
# so we may as well bail out before actual rsync happens.
if openpgp_env is not None and self.repo.sync_openpgp_key_path is not None:
try:
out.einfo('Using keys from %s' %
(self.repo.sync_openpgp_key_path, ))
with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
openpgp_env.import_key(f)
out.ebegin('Refreshing keys from keyserver')
retry_decorator = self._key_refresh_retry_decorator()
if retry_decorator is None:
openpgp_env.refresh_keys()
else:
loop = global_event_loop()
func_coroutine = functools.partial(
loop.run_in_executor, None,
openpgp_env.refresh_keys)
decorated_func = retry_decorator(func_coroutine)
loop.run_until_complete(decorated_func())
out.eend(0)
except (GematoException, TimeoutError) as e:
writemsg_level(
"!!! Manifest verification impossible due to keyring problem:\n%s\n"
% (e, ),
level=logging.ERROR,
noiselevel=-1)
return (1, False)
# Real local timestamp file.
self.servertimestampfile = os.path.join(self.repo.location,
"metadata",
"timestamp.chk")
content = portage.util.grabfile(self.servertimestampfile)
timestamp = 0
if content:
try:
timestamp = time.mktime(
time.strptime(content[0], TIMESTAMP_FORMAT))
except (OverflowError, ValueError):
pass
del content
try:
self.rsync_initial_timeout = \
int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
except ValueError:
self.rsync_initial_timeout = 15
try:
maxretries = int(self.settings["PORTAGE_RSYNC_RETRIES"])
except SystemExit as e:
raise # Needed else can't exit
except:
maxretries = -1 #default number of retries
if syncuri.startswith("file://"):
self.proto = "file"
dosyncuri = syncuri[7:]
unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
dosyncuri, timestamp, opts)
self._process_exitcode(exitcode, dosyncuri, out, 1)
return (exitcode, updatecache_flg)
retries = 0
try:
self.proto, user_name, hostname, port = re.split(
r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
syncuri,
maxsplit=4)[1:5]
except ValueError:
writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri,
noiselevel=-1,
level=logging.ERROR)
return (1, False)
self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
if port is None:
port = ""
if user_name is None:
user_name = ""
if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
getaddrinfo_host = hostname
else:
# getaddrinfo needs the brackets stripped
getaddrinfo_host = hostname[1:-1]
updatecache_flg = False
all_rsync_opts = set(self.rsync_opts)
all_rsync_opts.update(self.extra_rsync_opts)
family = socket.AF_UNSPEC
if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
family = socket.AF_INET
elif socket.has_ipv6 and \
("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
family = socket.AF_INET6
addrinfos = None
uris = []
try:
addrinfos = getaddrinfo_validate(
socket.getaddrinfo(getaddrinfo_host, None, family,
socket.SOCK_STREAM))
except socket.error as e:
writemsg_level("!!! getaddrinfo failed for '%s': %s\n" %
(_unicode_decode(hostname), _unicode(e)),
noiselevel=-1,
level=logging.ERROR)
if addrinfos:
AF_INET = socket.AF_INET
AF_INET6 = None
if socket.has_ipv6:
AF_INET6 = socket.AF_INET6
ips_v4 = []
ips_v6 = []
for addrinfo in addrinfos:
if addrinfo[0] == AF_INET:
ips_v4.append("%s" % addrinfo[4][0])
elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
# IPv6 addresses need to be enclosed in square brackets
ips_v6.append("[%s]" % addrinfo[4][0])
random.shuffle(ips_v4)
random.shuffle(ips_v6)
# Give priority to the address family that
# getaddrinfo() returned first.
if AF_INET6 is not None and addrinfos and \
addrinfos[0][0] == AF_INET6:
ips = ips_v6 + ips_v4
else:
ips = ips_v4 + ips_v6
for ip in ips:
uris.append(
syncuri.replace(
"//" + user_name + hostname + port + "/",
"//" + user_name + ip + port + "/", 1))
if not uris:
# With some configurations we need to use the plain hostname
# rather than try to resolve the ip addresses (bug #340817).
uris.append(syncuri)
# reverse, for use with pop()
uris.reverse()
uris_orig = uris[:]
effective_maxretries = maxretries
if effective_maxretries < 0:
effective_maxretries = len(uris) - 1
local_state_unchanged = True
while (1):
if uris:
dosyncuri = uris.pop()
elif maxretries < 0 or retries > maxretries:
writemsg("!!! Exhausted addresses for %s\n" %
_unicode_decode(hostname),
noiselevel=-1)
return (1, False)
else:
uris.extend(uris_orig)
dosyncuri = uris.pop()
if (retries == 0):
if "--ask" in opts:
uq = UserQuery(opts)
if uq.query("Do you want to sync your Portage tree " + \
"with the mirror at\n" + blue(dosyncuri) + bold("?"),
enter_invalid) == "No":
print()
print("Quitting.")
print()
sys.exit(128 + signal.SIGINT)
self.logger(self.xterm_titles,
">>> Starting rsync with " + dosyncuri)
if "--quiet" not in opts:
print(">>> Starting rsync with " + dosyncuri + "...")
else:
self.logger(self.xterm_titles,
">>> Starting retry %d of %d with %s" % \
(retries, effective_maxretries, dosyncuri))
writemsg_stdout(
"\n\n>>> Starting retry %d of %d with %s\n" % \
(retries, effective_maxretries, dosyncuri), noiselevel=-1)
if dosyncuri.startswith('ssh://'):
dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
dosyncuri, timestamp, opts)
if not unchanged:
local_state_unchanged = False
if is_synced:
break
retries = retries + 1
if maxretries < 0 or retries <= maxretries:
print(">>> Retrying...")
else:
# over retries
# exit loop
exitcode = EXCEEDED_MAX_RETRIES
break
self._process_exitcode(exitcode, dosyncuri, out, maxretries)
# if synced successfully, verify now
if exitcode == 0 and self.verify_metamanifest:
if gemato is None:
writemsg_level(
"!!! Unable to verify: gemato-11.0+ is required\n",
level=logging.ERROR,
noiselevel=-1)
exitcode = 127
else:
try:
# we always verify the Manifest signature, in case
# we had to deal with key revocation case
m = gemato.recursiveloader.ManifestRecursiveLoader(
os.path.join(self.repo.location, 'Manifest'),
verify_openpgp=True,
openpgp_env=openpgp_env,
max_jobs=self.verify_jobs)
if not m.openpgp_signed:
raise RuntimeError(
'OpenPGP signature not found on Manifest')
ts = m.find_timestamp()
if ts is None:
raise RuntimeError(
'Timestamp not found in Manifest')
if (self.max_age != 0
and (datetime.datetime.utcnow() - ts.ts).days >
self.max_age):
out.ewarn(
'Manifest is over %d days old, this is suspicious!'
% (self.max_age, ))
out.ewarn(
'You may want to try using another mirror and/or reporting this one:'
)
out.ewarn(' %s' % (dosyncuri, ))
out.ewarn('')
out.einfo('Manifest timestamp: %s UTC' % (ts.ts, ))
out.einfo('Valid OpenPGP signature found:')
out.einfo(
'- primary key: %s' %
(m.openpgp_signature.primary_key_fingerprint))
out.einfo('- subkey: %s' %
(m.openpgp_signature.fingerprint))
out.einfo('- timestamp: %s UTC' %
(m.openpgp_signature.timestamp))
# if nothing has changed, skip the actual Manifest
# verification
if not local_state_unchanged:
out.ebegin('Verifying %s' % (self.repo.location, ))
m.assert_directory_verifies()
out.eend(0)
except GematoException as e:
writemsg_level(
"!!! Manifest verification failed:\n%s\n" % (e, ),
level=logging.ERROR,
noiselevel=-1)
exitcode = 1
return (exitcode, updatecache_flg)
finally:
if openpgp_env is not None:
openpgp_env.close()
def _create_use_string(conf, name, cur_iuse, iuse_forced, cur_use, old_iuse,
old_use, is_new, feature_flags, reinst_flags):
if not conf.print_use_string:
return ""
enabled = []
if conf.alphabetical:
disabled = enabled
removed = enabled
else:
disabled = []
removed = []
cur_iuse = set(cur_iuse)
enabled_flags = cur_iuse.intersection(cur_use)
removed_iuse = set(old_iuse).difference(cur_iuse)
any_iuse = cur_iuse.union(old_iuse)
any_iuse = list(any_iuse)
any_iuse.sort()
for flag in any_iuse:
flag_str = None
isEnabled = False
reinst_flag = reinst_flags and flag in reinst_flags
if flag in enabled_flags:
isEnabled = True
if is_new or flag in old_use and \
(conf.all_flags or reinst_flag):
flag_str = red(flag)
elif flag not in old_iuse:
flag_str = yellow(flag) + "%*"
elif flag not in old_use:
flag_str = green(flag) + "*"
elif flag in removed_iuse:
if conf.all_flags or reinst_flag:
flag_str = yellow("-" + flag) + "%"
if flag in old_use:
flag_str += "*"
flag_str = "(" + flag_str + ")"
removed.append(flag_str)
continue
else:
if is_new or flag in old_iuse and \
flag not in old_use and \
(conf.all_flags or reinst_flag):
flag_str = blue("-" + flag)
elif flag not in old_iuse:
flag_str = yellow("-" + flag)
if flag not in iuse_forced:
flag_str += "%"
elif flag in old_use:
flag_str = green("-" + flag) + "*"
if flag_str:
if flag in feature_flags:
flag_str = "{" + flag_str + "}"
elif flag in iuse_forced:
flag_str = "(" + flag_str + ")"
if isEnabled:
enabled.append(flag_str)
else:
disabled.append(flag_str)
if conf.alphabetical:
ret = " ".join(enabled)
else:
ret = " ".join(enabled + disabled + removed)
if ret:
ret = '%s="%s" ' % (name, ret)
return ret
def main(settings=None, logger=None):
"""Main program operation method....
@param settings: dict. defaults to settings.DEFAULTS
@param logger: python logging module defaults to init_logger(settings)
@return boolean success/failure
"""
if settings is None:
print("NO Input settings, using defaults...")
settings = DEFAULTS.copy()
if logger is None:
logger = init_logger(settings)
_libs_to_check = settings['library']
if not settings['stdout'].isatty() or settings['nocolor']:
nocolor()
logger.warning(blue(' * ') +
yellow('This is the new python coded version'))
logger.warning(blue(' * ') +
yellow('Please report any bugs found using it.'))
logger.warning(blue(' * ') +
yellow('The original revdep-rebuild script is '
'installed as revdep-rebuild.sh'))
logger.warning(blue(' * ') +
yellow('Please file bugs at: '
'https://bugs.gentoo.org/'))
if os.getuid() != 0 and not settings['PRETEND']:
logger.warning(blue(' * ') +
yellow('You are not root, adding --pretend to portage options'))
settings['PRETEND'] = True
logger.debug("\tmain(), _libs_to_check = %s" % str(_libs_to_check))
if settings['USE_TMP_FILES'] \
and check_temp_files(settings['DEFAULT_TMP_DIR'], logger=logger):
libraries, la_libraries, libraries_links, binaries = read_cache(
settings['DEFAULT_TMP_DIR'])
assigned, orphaned = analyse(
settings=settings,
logger=logger,
libraries=libraries,
la_libraries=la_libraries,
libraries_links=libraries_links,
binaries=binaries,
_libs_to_check=_libs_to_check)
else:
assigned, orphaned = analyse(settings, logger, _libs_to_check=_libs_to_check)
if not assigned and not orphaned:
logger.warning('\n' + bold('Your system is consistent'))
# return the correct exit code
return 0
elif orphaned:
# blank line for beter visibility of the following lines
logger.warning('')
if settings['library']:
logger.warning(red(' !!! Dependant orphaned files: ') +
bold('No installed package was found for the following:'))
else:
logger.warning(red(' !!! Broken orphaned files: ') +
bold('No installed package was found for the following:'))
for filename in orphaned:
logger.warning(red('\t* ') + filename)
success = rebuild(logger, assigned, settings)
logger.debug("rebuild return code = %i" %success)
return success
def swig_generate(self):
"build all available modules"
def quote(s):
return '"' + s + '"'
def win_quote(s):
if sys.platform == 'win32':
return '"' + s + '"'
return s
for module in self.MODULES:
module_name = self.MODULES[module][0]
config_cmd = self.MODULES[module][1]
module_pkg_name = self.MODULES[module][2]
mod_hack_name = self.MODULES[module][3]
mod_out_prefix = module_pkg_name.replace('.', os.sep) + module
if sys.platform == "_win32":
INCLUDE_DIR = os.path.join(os.getenv("COINDIR"), "include")
CPP_FLAGS = "-I" + quote(INCLUDE_DIR) + " " + \
"-I" + quote(os.path.join(os.getenv("COINDIR"), "include", "Inventor", "annex")) + \
" /DCOIN_DLL /wd4244 /wd4049"
# aquire highest non-debug Coin library version
try:
LDFLAGS_LIBS = quote(
max(
glob.glob(
os.path.join(os.getenv("COINDIR"), "lib",
"coin?.lib")))) + " "
# with cmake the coin library is named Coin4.lib
except ValueError:
LDFLAGS_LIBS = quote(
max(
glob.glob(
os.path.join(os.getenv("COINDIR"), "lib",
"Coin?.lib")))) + " "
if module == "sowin":
CPP_FLAGS += " /DSOWIN_DLL"
LDFLAGS_LIBS += quote(
os.path.join(os.getenv("COINDIR"), "lib",
"sowin1.lib"))
elif module == "soqt":
CPP_FLAGS += " -I" + '"' + os.getenv(
"QTDIR") + "\\include\" /DSOQT_DLL"
if os.path.isdir(os.getenv("QTDIR") + "\\include\Qt\""):
CPP_FLAGS += " -I" + '"' + os.getenv(
"QTDIR") + "\\include\Qt\""
LDFLAGS_LIBS += os.path.join(os.getenv("COINDIR"),
"lib", "soqt1.lib") + " "
else:
# workaround for conda qt4:
CPP_FLAGS += " -I" + '"' + os.getenv(
"QTDIR") + "\\include\qt\Qt\""
CPP_FLAGS += " -I" + '"' + os.getenv(
"QTDIR") + "\\include\qt\""
LDFLAGS_LIBS += os.path.join(os.getenv("COINDIR"),
"lib", "SoQt.lib") + " "
else:
INCLUDE_DIR = self.cmake_config_dict[config_cmd +
'_INCLUDE_DIR']
LIB_DIR = self.cmake_config_dict[config_cmd + '_LIB_DIR']
CPP_FLAGS = ' -I' + win_quote(INCLUDE_DIR)
CPP_FLAGS += ' -I' + win_quote(
os.path.join(INCLUDE_DIR, 'Inventor', 'annex'))
if sys.platform == 'win32':
CPP_FLAGS += " /DCOIN_DLL /wd4244 /wd4049"
LDFLAGS_LIBS = quote(
max(glob.glob(os.path.join(LIB_DIR,
"Coin?.lib")))) + " "
else:
CPP_FLAGS += " -Wno-unused -Wno-maybe-uninitialized"
LDFLAGS_LIBS = ' -L' + self.cmake_config_dict[config_cmd +
'_LIB_DIR']
if module == "soqt":
CPP_FLAGS += ' -I' + win_quote(
self.QTINFO.getHeadersPath())
CPP_FLAGS += ' -I' + win_quote(
os.path.join(self.QTINFO.getHeadersPath(), 'QtCore'))
CPP_FLAGS += ' -I' + win_quote(
os.path.join(self.QTINFO.getHeadersPath(), 'QtGui'))
CPP_FLAGS += ' -I' + win_quote(
os.path.join(self.QTINFO.getHeadersPath(), 'QtOpenGL'))
CPP_FLAGS += ' -I' + win_quote(
os.path.join(self.QTINFO.getHeadersPath(),
'QtWidgets'))
if sys.platform == 'win32':
LDFLAGS_LIBS += " " + quote(
max(glob.glob(os.path.join(LIB_DIR,
"SoQt?.lib")))) + " "
CPP_FLAGS += " /DSOQT_DLL"
else:
LDFLAGS_LIBS += ' -lSoQt'
if module == "coin":
if sys.platform == 'win32':
pass
else:
LDFLAGS_LIBS += ' -lCoin'
if not os.path.isfile(mod_out_prefix + "_wrap.cpp"):
print(
red("\n=== Generating %s_wrap.cpp for %s ===\n" %
(mod_out_prefix, module)))
print(
blue(self.SWIG + " " + self.SWIG_SUPPRESS_WARNINGS + " " +
self.SWIG_PARAMS %
(INCLUDE_DIR, self.CXX_INCS, mod_out_prefix, module)))
if os.system(self.SWIG + " " + self.SWIG_SUPPRESS_WARNINGS +
" " + self.SWIG_PARAMS %
(INCLUDE_DIR, self.CXX_INCS, mod_out_prefix,
mod_hack_name)):
print(
red("SWIG did not generate wrappers successfully! ** Aborting **"
))
sys.exit(1)
else:
print(
red("=== %s_wrap.cpp for %s already exists! ===" %
(mod_out_prefix, module_pkg_name + module)))
self.ext_modules.append(
Extension(
module_name, [mod_out_prefix + "_wrap.cpp"],
extra_compile_args=(self.CXX_INCS + CPP_FLAGS).split(),
extra_link_args=(self.CXX_LIBS + LDFLAGS_LIBS).split()))
def update(self):
'''Internal update function which performs the transfer'''
opts = self.options.get('emerge_config').opts
self.usersync_uid = self.options.get('usersync_uid', None)
enter_invalid = '--ask-enter-invalid' in opts
out = portage.output.EOutput()
syncuri = self.repo.sync_uri
vcs_dirs = frozenset(VCS_DIRS)
vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
for vcs_dir in vcs_dirs:
writemsg_level(("!!! %s appears to be under revision " + \
"control (contains %s).\n!!! Aborting rsync sync.\n") % \
(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
return (1, False)
self.timeout = 180
rsync_opts = []
if self.settings["PORTAGE_RSYNC_OPTS"] == "":
rsync_opts = self._set_rsync_defaults()
else:
rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
self.extra_rsync_opts = list()
if self.repo.module_specific_options.get('sync-rsync-extra-opts'):
self.extra_rsync_opts.extend(
portage.util.shlex_split(
self.repo.module_specific_options['sync-rsync-extra-opts'])
)
# Real local timestamp file.
self.servertimestampfile = os.path.join(self.repo.location, "metadata",
"timestamp.chk")
content = portage.util.grabfile(self.servertimestampfile)
timestamp = 0
if content:
try:
timestamp = time.mktime(
time.strptime(content[0], TIMESTAMP_FORMAT))
except (OverflowError, ValueError):
pass
del content
try:
self.rsync_initial_timeout = \
int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
except ValueError:
self.rsync_initial_timeout = 15
try:
maxretries = int(self.settings["PORTAGE_RSYNC_RETRIES"])
except SystemExit as e:
raise # Needed else can't exit
except:
maxretries = -1 #default number of retries
if syncuri.startswith("file://"):
self.proto = "file"
dosyncuri = syncuri[6:]
is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
self._process_exitcode(exitcode, dosyncuri, out, 1)
return (exitcode, exitcode == os.EX_OK)
retries = 0
try:
self.proto, user_name, hostname, port = re.split(
r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
syncuri,
maxsplit=4)[1:5]
except ValueError:
writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri,
noiselevel=-1,
level=logging.ERROR)
return (1, False)
self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
if port is None:
port = ""
if user_name is None:
user_name = ""
if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
getaddrinfo_host = hostname
else:
# getaddrinfo needs the brackets stripped
getaddrinfo_host = hostname[1:-1]
updatecache_flg = True
all_rsync_opts = set(self.rsync_opts)
all_rsync_opts.update(self.extra_rsync_opts)
family = socket.AF_UNSPEC
if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
family = socket.AF_INET
elif socket.has_ipv6 and \
("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
family = socket.AF_INET6
addrinfos = None
uris = []
try:
addrinfos = getaddrinfo_validate(
socket.getaddrinfo(getaddrinfo_host, None, family,
socket.SOCK_STREAM))
except socket.error as e:
writemsg_level("!!! getaddrinfo failed for '%s': %s\n" %
(_unicode_decode(hostname), _unicode(e)),
noiselevel=-1,
level=logging.ERROR)
if addrinfos:
AF_INET = socket.AF_INET
AF_INET6 = None
if socket.has_ipv6:
AF_INET6 = socket.AF_INET6
ips_v4 = []
ips_v6 = []
for addrinfo in addrinfos:
if addrinfo[0] == AF_INET:
ips_v4.append("%s" % addrinfo[4][0])
elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
# IPv6 addresses need to be enclosed in square brackets
ips_v6.append("[%s]" % addrinfo[4][0])
random.shuffle(ips_v4)
random.shuffle(ips_v6)
# Give priority to the address family that
# getaddrinfo() returned first.
if AF_INET6 is not None and addrinfos and \
addrinfos[0][0] == AF_INET6:
ips = ips_v6 + ips_v4
else:
ips = ips_v4 + ips_v6
for ip in ips:
uris.append(
syncuri.replace("//" + user_name + hostname + port + "/",
"//" + user_name + ip + port + "/", 1))
if not uris:
# With some configurations we need to use the plain hostname
# rather than try to resolve the ip addresses (bug #340817).
uris.append(syncuri)
# reverse, for use with pop()
uris.reverse()
effective_maxretries = maxretries
if effective_maxretries < 0:
effective_maxretries = len(uris) - 1
while (1):
if uris:
dosyncuri = uris.pop()
else:
writemsg("!!! Exhausted addresses for %s\n" %
_unicode_decode(hostname),
noiselevel=-1)
return (1, False)
if (retries == 0):
if "--ask" in opts:
uq = UserQuery(opts)
if uq.query("Do you want to sync your Portage tree " + \
"with the mirror at\n" + blue(dosyncuri) + bold("?"),
enter_invalid) == "No":
print()
print("Quitting.")
print()
sys.exit(128 + signal.SIGINT)
self.logger(self.xterm_titles,
">>> Starting rsync with " + dosyncuri)
if "--quiet" not in opts:
print(">>> Starting rsync with " + dosyncuri + "...")
else:
self.logger(self.xterm_titles,
">>> Starting retry %d of %d with %s" % \
(retries, effective_maxretries, dosyncuri))
writemsg_stdout(
"\n\n>>> Starting retry %d of %d with %s\n" % \
(retries, effective_maxretries, dosyncuri), noiselevel=-1)
if dosyncuri.startswith('ssh://'):
dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
if is_synced:
break
retries = retries + 1
if maxretries < 0 or retries <= maxretries:
print(">>> Retrying...")
else:
# over retries
# exit loop
updatecache_flg = False
exitcode = EXCEEDED_MAX_RETRIES
break
self._process_exitcode(exitcode, dosyncuri, out, maxretries)
return (exitcode, updatecache_flg)
def check_with_cmake(self):
dirname = os.path.dirname(__file__)
cmake_command = ['cmake', dirname]
try:
cmake_command += ['-G', os.environ['GENERATOR']]
except KeyError:
pass
print(yellow('calling: ' + cmake_command[0] + ' ' + cmake_command[1]))
cmake = subprocess.Popen(cmake_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
cmake_out, cmake_err = cmake.communicate()
coin_vars = [
'COIN_FOUND', 'COIN_VERSION', 'COIN_INCLUDE_DIR', 'COIN_LIB_DIR'
]
soqt_vars = [
'SOQT_FOUND', 'SOQT_VERSION', 'SOQT_INCLUDE_DIR', 'SOQT_LIB_DIR'
]
config_dict = {}
print(yellow(cmake_out.decode("utf-8")))
print(red(cmake_err.decode("utf-8")))
if cmake.returncode == 0:
for line in cmake_out.decode("utf-8").split("\n"):
for var in coin_vars + soqt_vars:
if var in line:
line = (line.replace('-- ' + var,
'').replace(': ',
'').replace('\n', ''))
if "INCLUDE" in var:
line = line.replace(';', '\" -I\"')
config_dict[var] = line
print(yellow('\nchecking for COIN via cmake'))
for key in coin_vars:
if key in config_dict:
print(blue(key + ': ' + config_dict[key]))
print(yellow('\nchecking for SOQT via cmake'))
for key in soqt_vars:
if key in config_dict:
print(blue(key + ': ' + config_dict[key]))
if config_dict.get('SOQT_FOUND', 'false') == 'false':
pivy_build.MODULES.pop('soqt')
print(red("\ndisable soqt, because cmake couldn't find it"))
else:
try:
import qtinfo
self.QTINFO = qtinfo.QtInfo()
except Exception as e:
import traceback
print(
red("\ndisable soqt, because there was a problem running qtinfo (needs qmake)"
))
print(red("-" * 60))
print(red(traceback.print_exc()))
print(red("-" * 60))
pivy_build.MODULES.pop('soqt')
self.cmake_config_dict = config_dict
if self.cmake_config_dict.get('COIN_FOUND', 'false') == 'false':
raise (RuntimeError(
'coin was not found, but you need coin to build pivy'))
def main(settings=None, logger=None):
"""Main program operation method....
@param settings: dict. defaults to settings.DEFAULTS
@param logger: python logging module defaults to init_logger(settings)
@return boolean success/failure
"""
if settings is None:
print("NO Input settings, using defaults...")
settings = DEFAULTS.copy()
if logger is None:
logger = init_logger(settings)
_libs_to_check = settings['library']
if not settings['stdout'].isatty() or settings['nocolor']:
nocolor()
#TODO: Development warning
logger.warn(blue(' * ') +
yellow('This is a development version, '
'so it may not work correctly'))
logger.warn(blue(' * ') +
yellow('The original revdep-rebuild script is '
'installed as revdep-rebuild.sh'))
if os.getuid() != 0 and not settings['PRETEND']:
logger.warn(blue(' * ') +
yellow('You are not root, adding --pretend to portage options'))
settings['PRETEND'] = True
if settings['library']:
logger.warn(green(' * ') +
"Looking for libraries: %s" % (bold(', '.join(settings['library']))))
if settings['USE_TMP_FILES'] \
and check_temp_files(settings['DEFAULT_TMP_DIR'], logger=logger):
libraries, la_libraries, libraries_links, binaries = read_cache(
settings['DEFAULT_TMP_DIR'])
assigned = analyse(
settings=settings,
logger=logger,
libraries=libraries,
la_libraries=la_libraries,
libraries_links=libraries_links,
binaries=binaries,
_libs_to_check=_libs_to_check)
else:
assigned = analyse(settings, logger, _libs_to_check=_libs_to_check)
if not assigned:
logger.warn('\n' + bold('Your system is consistent'))
# return the correct exit code
return 0
has_masked = False
tmp = []
for ebuild in assigned:
if get_masking_status(ebuild):
has_masked = True
logger.warn('!!! ' + red('All ebuilds that could satisfy: ') +
green(ebuild) + red(' have been masked'))
else:
tmp.append(ebuild)
assigned = tmp
if has_masked:
logger.info(red(' * ') +
'Unmask all ebuild(s) listed above and call revdep-rebuild '
'again or manually emerge given packages.')
success = rebuild(logger, assigned, settings)
logger.debug("rebuild return code = %i" %success)
return success
def useflag(string, enabled=True): """Returns a USE flag string.""" return output.red(string) if enabled else output.blue(string)
def check_python_version(self):
"check the Python version"
print blue("Python version...%s" % sys.version.split(" ")[0])
if int(sys.version[0]) < 2:
print red("Pivy only works with Python versions >= 2.0.")
sys.exit(1)
sys.stdout.flush()
def formatSize(x):
x = int(x)
if x < 1024:
return str(x) + ' B'
if x < 1024 * 1024:
return '%.2f KB' % (float(x) / 1024)
return '%.2f MB' % (float(x) / (1024.0 * 1024.0))
packages = {}
total_size = 0
print output.blue('Counting... ')
counter = 0
for category in sorted(os.listdir('/var/db/pkg/')):
print output.green(' *'), category, ' ',
update_spinner()
for package in sorted(os.listdir('/var/db/pkg/' + category)):
fh = file('/var/db/pkg/' + category + '/' + package + '/CONTENTS')
content = fh.readlines()
fh.close()
size = 0
for entity in content:
def useflag(string, enabled=True):
"""Returns a USE flag string."""
return output.red(string) if enabled else output.blue(string)
def analyse(
settings, logger, libraries=None, la_libraries=None, libraries_links=None, binaries=None, _libs_to_check=set()
):
"""Main program body. It will collect all info and determine the
pkgs needing rebuilding.
@param logger: logger used for logging messages, instance of logging.Logger
class. Can be logging (RootLogger).
@param _libs_to_check Libraries that need to be checked only
@rtype list: list of pkgs that need rebuilding
"""
if libraries and la_libraries and libraries_links and binaries:
logger.info(blue(" * ") + bold("Found a valid cache, skipping collecting phase"))
else:
# TODO: add partial cache (for ex. only libraries)
# when found for some reason
logger.warn(green(" * ") + bold("Collecting system binaries and libraries"))
bin_dirs, lib_dirs = prepare_search_dirs(logger, settings)
masked_dirs, masked_files, ld = parse_revdep_config(settings["REVDEP_CONFDIR"])
lib_dirs.update(ld)
bin_dirs.update(ld)
masked_dirs.update(["/lib/modules", "/lib32/modules", "/lib64/modules"])
logger.info(green(" * ") + bold("Collecting dynamic linking informations"))
libraries, la_libraries, libraries_links, symlink_pairs = collect_libraries_from_dir(
lib_dirs, masked_dirs, logger
)
binaries = collect_binaries_from_dir(bin_dirs, masked_dirs, logger)
if settings["USE_TMP_FILES"]:
save_cache(
logger=logger,
to_save={
"libraries": libraries,
"la_libraries": la_libraries,
"libraries_links": libraries_links,
"binaries": binaries,
},
temp_path=settings["DEFAULT_TMP_DIR"],
)
logger.debug(
"Found "
+ str(len(libraries))
+ " libraries (+"
+ str(len(libraries_links))
+ " symlinks) and "
+ str(len(binaries))
+ " binaries"
)
logger.warn(green(" * ") + bold("Checking dynamic linking consistency"))
logger.debug(
"Search for " + str(len(binaries) + len(libraries)) + " within " + str(len(libraries) + len(libraries_links))
)
libs_and_bins = libraries + binaries
found_libs = []
dependencies = []
if _libs_to_check:
nltc = []
for ltc in _libs_to_check:
if os.path.isfile(ltc):
ltc = scan(["-nBSF", "%S"], [ltc], settings["CMD_MAX_ARGS"])[0].split()[0]
nltc += [ltc]
_libs_to_check = nltc
_bits, linkg = platform.architecture()
if _bits.startswith("32"):
bits = 32
elif _bits.startswith("64"):
bits = 64
broken = []
for av_bits in glob.glob("/lib[0-9]*") or ("/lib32",):
bits = int(av_bits[4:])
_libraries = libraries + libraries_links
found_libs, dependencies = prepare_checks(libs_and_bins, _libraries, bits, settings["CMD_MAX_ARGS"])
broken = find_broken(found_libs, _libraries, _libs_to_check)
bits /= 2
bits = int(bits)
broken_la = extract_dependencies_from_la(la_libraries, libraries + libraries_links, _libs_to_check, logger)
broken_pathes = main_checks(found_libs, broken, dependencies, logger)
broken_pathes += broken_la
logger.warn(green(" * ") + bold("Assign files to packages"))
return assign_packages(broken_pathes, logger, settings)
def check_python_version(self):
"check the Python version"
print(blue("Python version...%s" % sys.version.split(" ")[0]))
if int(sys.version[0]) < 2:
print(red("Pivy only works with Python versions >= 2.0."))
sys.exit(1)
def update(self):
"""Internal update function which performs the transfer"""
opts = self.options.get("emerge_config").opts
self.usersync_uid = self.options.get("usersync_uid", None)
enter_invalid = "--ask-enter-invalid" in opts
quiet = "--quiet" in opts
out = portage.output.EOutput(quiet=quiet)
syncuri = self.repo.sync_uri
if self.repo.module_specific_options.get("sync-rsync-vcs-ignore",
"false").lower() in ("true",
"yes"):
vcs_dirs = ()
else:
vcs_dirs = frozenset(VCS_DIRS)
vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
for vcs_dir in vcs_dirs:
writemsg_level(
("!!! %s appears to be under revision " +
"control (contains %s).\n!!! Aborting rsync sync "
'(override with "sync-rsync-vcs-ignore = true" in repos.conf).\n'
) % (self.repo.location, vcs_dir),
level=logging.ERROR,
noiselevel=-1,
)
return (1, False)
self.timeout = 180
rsync_opts = []
if self.settings["PORTAGE_RSYNC_OPTS"] == "":
rsync_opts = self._set_rsync_defaults()
else:
rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
self.extra_rsync_opts = list()
if self.repo.module_specific_options.get("sync-rsync-extra-opts"):
self.extra_rsync_opts.extend(
portage.util.shlex_split(
self.repo.module_specific_options["sync-rsync-extra-opts"])
)
exitcode = 0
verify_failure = False
# Process GLEP74 verification options.
# Default verification to 'no'; it's enabled for ::gentoo
# via default repos.conf though.
self.verify_metamanifest = self.repo.module_specific_options.get(
"sync-rsync-verify-metamanifest", "no").lower() in ("yes", "true")
# Support overriding job count.
self.verify_jobs = self.repo.module_specific_options.get(
"sync-rsync-verify-jobs", None)
if self.verify_jobs is not None:
try:
self.verify_jobs = int(self.verify_jobs)
if self.verify_jobs < 0:
raise ValueError(self.verify_jobs)
except ValueError:
writemsg_level(
"!!! sync-rsync-verify-jobs not a positive integer: %s\n" %
(self.verify_jobs, ),
level=logging.WARNING,
noiselevel=-1,
)
self.verify_jobs = None
else:
if self.verify_jobs == 0:
# Use the apparent number of processors if gemato
# supports it.
self.verify_jobs = None
# Support overriding max age.
self.max_age = self.repo.module_specific_options.get(
"sync-rsync-verify-max-age", "")
if self.max_age:
try:
self.max_age = int(self.max_age)
if self.max_age < 0:
raise ValueError(self.max_age)
except ValueError:
writemsg_level(
"!!! sync-rsync-max-age must be a non-negative integer: %s\n"
% (self.max_age, ),
level=logging.WARNING,
noiselevel=-1,
)
self.max_age = 0
else:
self.max_age = 0
openpgp_env = None
if self.verify_metamanifest and gemato is not None:
# Use isolated environment if key is specified,
# system environment otherwise
openpgp_env = self._get_openpgp_env(
self.repo.sync_openpgp_key_path)
try:
# Load and update the keyring early. If it fails, then verification
# will not be performed and the user will have to fix it and try again,
# so we may as well bail out before actual rsync happens.
if openpgp_env is not None and self.repo.sync_openpgp_key_path is not None:
try:
out.einfo("Using keys from %s" %
(self.repo.sync_openpgp_key_path, ))
with io.open(self.repo.sync_openpgp_key_path, "rb") as f:
openpgp_env.import_key(f)
self._refresh_keys(openpgp_env)
except (GematoException, asyncio.TimeoutError) as e:
writemsg_level(
"!!! Manifest verification impossible due to keyring problem:\n%s\n"
% (e, ),
level=logging.ERROR,
noiselevel=-1,
)
return (1, False)
# Real local timestamp file.
self.servertimestampfile = os.path.join(self.repo.location,
"metadata",
"timestamp.chk")
content = portage.util.grabfile(self.servertimestampfile)
timestamp = 0
if content:
try:
timestamp = time.mktime(
time.strptime(content[0], TIMESTAMP_FORMAT))
except (OverflowError, ValueError):
pass
del content
try:
self.rsync_initial_timeout = int(
self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
except ValueError:
self.rsync_initial_timeout = 15
try:
maxretries = int(self.settings["PORTAGE_RSYNC_RETRIES"])
except SystemExit as e:
raise # Needed else can't exit
except:
maxretries = -1 # default number of retries
if syncuri.startswith("file://"):
self.proto = "file"
dosyncuri = syncuri[7:]
unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
dosyncuri, timestamp, opts)
self._process_exitcode(exitcode, dosyncuri, out, 1)
if exitcode == 0:
if unchanged:
self.repo_storage.abort_update()
else:
self.repo_storage.commit_update()
self.repo_storage.garbage_collection()
return (exitcode, updatecache_flg)
retries = 0
try:
self.proto, user_name, hostname, port = re.split(
r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
syncuri,
maxsplit=4,
)[1:5]
except ValueError:
writemsg_level(
"!!! sync-uri is invalid: %s\n" % syncuri,
noiselevel=-1,
level=logging.ERROR,
)
return (1, False)
self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
if port is None:
port = ""
if user_name is None:
user_name = ""
if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
getaddrinfo_host = hostname
else:
# getaddrinfo needs the brackets stripped
getaddrinfo_host = hostname[1:-1]
updatecache_flg = False
all_rsync_opts = set(self.rsync_opts)
all_rsync_opts.update(self.extra_rsync_opts)
family = socket.AF_UNSPEC
if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
family = socket.AF_INET
elif socket.has_ipv6 and ("-6" in all_rsync_opts
or "--ipv6" in all_rsync_opts):
family = socket.AF_INET6
addrinfos = None
uris = []
if "RSYNC_PROXY" not in self.spawn_kwargs["env"]:
try:
addrinfos = getaddrinfo_validate(
socket.getaddrinfo(getaddrinfo_host, None, family,
socket.SOCK_STREAM))
except socket.error as e:
writemsg_level(
"!!! getaddrinfo failed for '%s': %s\n" %
(_unicode_decode(hostname), str(e)),
noiselevel=-1,
level=logging.ERROR,
)
if addrinfos:
AF_INET = socket.AF_INET
AF_INET6 = None
if socket.has_ipv6:
AF_INET6 = socket.AF_INET6
ips_v4 = []
ips_v6 = []
for addrinfo in addrinfos:
if addrinfo[0] == AF_INET:
ips_v4.append("%s" % addrinfo[4][0])
elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
# IPv6 addresses need to be enclosed in square brackets
ips_v6.append("[%s]" % addrinfo[4][0])
random.shuffle(ips_v4)
random.shuffle(ips_v6)
# Give priority to the address family that
# getaddrinfo() returned first.
if AF_INET6 is not None and addrinfos and addrinfos[0][
0] == AF_INET6:
ips = ips_v6 + ips_v4
else:
ips = ips_v4 + ips_v6
for ip in ips:
uris.append(
syncuri.replace(
"//" + user_name + hostname + port + "/",
"//" + user_name + ip + port + "/",
1,
))
if not uris:
# With some configurations we need to use the plain hostname
# rather than try to resolve the ip addresses (bug #340817).
uris.append(syncuri)
elif len(uris) == 1:
# Use the original hostname if it resolves to a single IP,
# since DNS lookup must occur in the rsync process for
# compatibility with things like proxychains that allocate
# a surrogate IP which is only valid within the current
# process.
uris = [syncuri]
# reverse, for use with pop()
uris.reverse()
uris_orig = uris[:]
effective_maxretries = maxretries
if effective_maxretries < 0:
effective_maxretries = len(uris) - 1
local_state_unchanged = True
while 1:
if uris:
dosyncuri = uris.pop()
elif maxretries < 0 or retries > maxretries:
writemsg(
"!!! Exhausted addresses for %s\n" %
_unicode_decode(hostname),
noiselevel=-1,
)
return (1, False)
else:
uris.extend(uris_orig)
dosyncuri = uris.pop()
if retries == 0:
if "--ask" in opts:
uq = UserQuery(opts)
if (uq.query(
"Do you want to sync your ebuild repository " +
"with the mirror at\n" + blue(dosyncuri) +
bold("?"),
enter_invalid,
) == "No"):
print()
print("Quitting.")
print()
sys.exit(128 + signal.SIGINT)
self.logger(self.xterm_titles,
">>> Starting rsync with " + dosyncuri)
if "--quiet" not in opts:
print(">>> Starting rsync with " + dosyncuri + "...")
else:
self.logger(
self.xterm_titles,
">>> Starting retry %d of %d with %s" %
(retries, effective_maxretries, dosyncuri),
)
writemsg_stdout(
"\n\n>>> Starting retry %d of %d with %s\n" %
(retries, effective_maxretries, dosyncuri),
noiselevel=-1,
)
if dosyncuri.startswith("ssh://"):
dosyncuri = dosyncuri[6:].replace("/", ":/", 1)
unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
dosyncuri, timestamp, opts)
if not unchanged:
local_state_unchanged = False
if is_synced:
break
retries = retries + 1
if maxretries < 0 or retries <= maxretries:
print(">>> Retrying...")
else:
# over retries
# exit loop
exitcode = EXCEEDED_MAX_RETRIES
break
self._process_exitcode(exitcode, dosyncuri, out, maxretries)
if local_state_unchanged:
# The quarantine download_dir is not intended to exist
# in this case, so refer gemato to the normal repository
# location.
download_dir = self.repo.location
else:
download_dir = self.download_dir
# if synced successfully, verify now
if exitcode == 0 and self.verify_metamanifest:
if gemato is None:
writemsg_level(
"!!! Unable to verify: gemato-14.5+ is required\n",
level=logging.ERROR,
noiselevel=-1,
)
exitcode = 127
else:
try:
# we always verify the Manifest signature, in case
# we had to deal with key revocation case
m = gemato.recursiveloader.ManifestRecursiveLoader(
os.path.join(download_dir, "Manifest"),
verify_openpgp=True,
openpgp_env=openpgp_env,
max_jobs=self.verify_jobs,
)
if not m.openpgp_signed:
raise RuntimeError(
"OpenPGP signature not found on Manifest")
ts = m.find_timestamp()
if ts is None:
raise RuntimeError(
"Timestamp not found in Manifest")
if (self.max_age != 0
and (datetime.datetime.utcnow() - ts.ts).days >
self.max_age):
out.quiet = False
out.ewarn(
"Manifest is over %d days old, this is suspicious!"
% (self.max_age, ))
out.ewarn(
"You may want to try using another mirror and/or reporting this one:"
)
out.ewarn(" %s" % (dosyncuri, ))
out.ewarn("")
out.quiet = quiet
out.einfo("Manifest timestamp: %s UTC" % (ts.ts, ))
out.einfo("Valid OpenPGP signature found:")
out.einfo(
"- primary key: %s" %
(m.openpgp_signature.primary_key_fingerprint))
out.einfo("- subkey: %s" %
(m.openpgp_signature.fingerprint))
out.einfo("- timestamp: %s UTC" %
(m.openpgp_signature.timestamp))
# if nothing has changed, skip the actual Manifest
# verification
if not local_state_unchanged:
out.ebegin("Verifying %s" % (download_dir, ))
m.assert_directory_verifies()
out.eend(0)
except GematoException as e:
writemsg_level(
"!!! Manifest verification failed:\n%s\n" % (e, ),
level=logging.ERROR,
noiselevel=-1,
)
exitcode = 1
verify_failure = True
if exitcode == 0 and not local_state_unchanged:
self.repo_storage.commit_update()
self.repo_storage.garbage_collection()
return (exitcode, updatecache_flg)
finally:
# Don't delete the update if verification failed, in case
# the cause needs to be investigated.
if not verify_failure:
self.repo_storage.abort_update()
if openpgp_env is not None:
openpgp_env.close()
def analyse(settings, logger, libraries=None, la_libraries=None,
libraries_links=None, binaries=None, _libs_to_check=None):
"""Main program body. It will collect all info and determine the
pkgs needing rebuilding.
@param logger: logger used for logging messages, instance of logging.Logger
class. Can be logging (RootLogger).
@param _libs_to_check Libraries that need to be checked only
@rtype list: list of pkgs that need rebuilding
"""
searchbits = set()
'''if _libs_to_check:
for lib in _libs_to_check:
if "lib64" in lib:
searchbits.add('64')
elif "lib32" in lib:
searchbits.add('32')
else:
_libs_to_check = set()'''
searchbits.update(['64', '32'])
masked_dirs, masked_files, ld = parse_revdep_config(settings['REVDEP_CONFDIR'])
masked_dirs.update([
'/lib/modules',
'/lib32/modules',
'/lib64/modules',
]
)
if '64' not in searchbits:
masked_dirs.update(['/lib64', '/usr/lib64'])
elif '32' not in searchbits:
masked_dirs.update(['/lib32', '/usr/lib32'])
all_masks = masked_dirs.copy()
all_masks.update(masked_files)
logger.debug("\tall_masks:")
for x in sorted(all_masks):
logger.debug('\t\t%s' % (x))
if libraries and la_libraries and libraries_links and binaries:
logger.info(blue(' * ') +
bold('Found a valid cache, skipping collecting phase'))
else:
#TODO: add partial cache (for ex. only libraries)
# when found for some reason
stime = current_milli_time()
logger.warning(green(' * ') +
bold('Collecting system binaries and libraries'))
bin_dirs, lib_dirs = prepare_search_dirs(logger, settings)
lib_dirs.update(ld)
bin_dirs.update(ld)
logger.debug('\tanalyse(), bin directories:')
for x in sorted(bin_dirs):
logger.debug('\t\t%s' % (x))
logger.debug('\tanalyse(), lib directories:')
for x in sorted(lib_dirs):
logger.debug('\t\t%s' % (x))
logger.debug('\tanalyse(), masked directories:')
for x in sorted(masked_dirs):
logger.debug('\t\t%s' % (x))
logger.debug('\tanalyse(), masked files:')
for x in sorted(masked_files):
logger.debug('\t\t%s' % (x))
ftime = current_milli_time()
logger.debug('\ttime to complete task: %d milliseconds' % (ftime-stime))
stime = current_milli_time()
logger.info(green(' * ') +
bold('Collecting dynamic linking informations'))
libraries, la_libraries, libraries_links = \
collect_libraries_from_dir(lib_dirs, all_masks, logger)
binaries = collect_binaries_from_dir(bin_dirs, all_masks, logger)
ftime = current_milli_time()
logger.debug('\ttime to complete task: %d milliseconds' % (ftime-stime))
if settings['USE_TMP_FILES']:
save_cache(logger=logger,
to_save={'libraries':libraries, 'la_libraries':la_libraries,
'libraries_links':libraries_links, 'binaries':binaries
},
temp_path=settings['DEFAULT_TMP_DIR']
)
logger.debug('\tanalyse(), Found %i libraries (+%i symlinks) and %i binaries' %
(len(libraries), len(libraries_links), len(binaries))
)
logger.info(green(' * ') + bold('Scanning files'))
libs_and_bins = libraries.union(binaries)
scanned_files = scan_files(libs_and_bins, settings['CMD_MAX_ARGS'],
logger, searchbits)
logger.warning(green(' * ') + bold('Checking dynamic linking consistency'))
logger.debug(
'\tanalyse(), Searching for %i libs, bins within %i libraries and links'
% (len(libs_and_bins), len(libraries)+len(libraries_links))
)
libcheck = LibCheck(scanned_files, logger, _libs_to_check, searchbits,
all_masks, masked_dirs)
broken_pathes = libcheck.process_results(libcheck.search())
broken_la = extract_dependencies_from_la(la_libraries,
libraries.union(libraries_links), _libs_to_check, logger)
broken_pathes += broken_la
if broken_pathes:
logger.warning(green(' * ') + bold('Assign files to packages'))
return assign_packages(broken_pathes, logger, settings)
return None, None # no need to assign anything
def update(self):
'''Internal update function which performs the transfer'''
opts = self.options.get('emerge_config').opts
self.usersync_uid = self.options.get('usersync_uid', None)
enter_invalid = '--ask-enter-invalid' in opts
quiet = '--quiet' in opts
out = portage.output.EOutput(quiet=quiet)
syncuri = self.repo.sync_uri
if self.repo.module_specific_options.get(
'sync-rsync-vcs-ignore', 'false').lower() == 'true':
vcs_dirs = ()
else:
vcs_dirs = frozenset(VCS_DIRS)
vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
for vcs_dir in vcs_dirs:
writemsg_level(("!!! %s appears to be under revision " + \
"control (contains %s).\n!!! Aborting rsync sync "
"(override with \"sync-rsync-vcs-ignore = true\" in repos.conf).\n") % \
(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
return (1, False)
self.timeout=180
rsync_opts = []
if self.settings["PORTAGE_RSYNC_OPTS"] == "":
rsync_opts = self._set_rsync_defaults()
else:
rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
self.extra_rsync_opts = list()
if self.repo.module_specific_options.get('sync-rsync-extra-opts'):
self.extra_rsync_opts.extend(portage.util.shlex_split(
self.repo.module_specific_options['sync-rsync-extra-opts']))
exitcode = 0
verify_failure = False
# Process GLEP74 verification options.
# Default verification to 'no'; it's enabled for ::gentoo
# via default repos.conf though.
self.verify_metamanifest = (
self.repo.module_specific_options.get(
'sync-rsync-verify-metamanifest', 'no') in ('yes', 'true'))
# Support overriding job count.
self.verify_jobs = self.repo.module_specific_options.get(
'sync-rsync-verify-jobs', None)
if self.verify_jobs is not None:
try:
self.verify_jobs = int(self.verify_jobs)
if self.verify_jobs < 0:
raise ValueError(self.verify_jobs)
except ValueError:
writemsg_level("!!! sync-rsync-verify-jobs not a positive integer: %s\n" % (self.verify_jobs,),
level=logging.WARNING, noiselevel=-1)
self.verify_jobs = None
else:
if self.verify_jobs == 0:
# Use the apparent number of processors if gemato
# supports it.
self.verify_jobs = None
# Support overriding max age.
self.max_age = self.repo.module_specific_options.get(
'sync-rsync-verify-max-age', '')
if self.max_age:
try:
self.max_age = int(self.max_age)
if self.max_age < 0:
raise ValueError(self.max_age)
except ValueError:
writemsg_level("!!! sync-rsync-max-age must be a non-negative integer: %s\n" % (self.max_age,),
level=logging.WARNING, noiselevel=-1)
self.max_age = 0
else:
self.max_age = 0
openpgp_env = None
if self.verify_metamanifest and gemato is not None:
# Use isolated environment if key is specified,
# system environment otherwise
if self.repo.sync_openpgp_key_path is not None:
openpgp_env = gemato.openpgp.OpenPGPEnvironment()
else:
openpgp_env = gemato.openpgp.OpenPGPSystemEnvironment()
try:
# Load and update the keyring early. If it fails, then verification
# will not be performed and the user will have to fix it and try again,
# so we may as well bail out before actual rsync happens.
if openpgp_env is not None and self.repo.sync_openpgp_key_path is not None:
try:
out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
openpgp_env.import_key(f)
self._refresh_keys(openpgp_env)
except (GematoException, asyncio.TimeoutError) as e:
writemsg_level("!!! Manifest verification impossible due to keyring problem:\n%s\n"
% (e,),
level=logging.ERROR, noiselevel=-1)
return (1, False)
# Real local timestamp file.
self.servertimestampfile = os.path.join(
self.repo.location, "metadata", "timestamp.chk")
content = portage.util.grabfile(self.servertimestampfile)
timestamp = 0
if content:
try:
timestamp = time.mktime(time.strptime(content[0],
TIMESTAMP_FORMAT))
except (OverflowError, ValueError):
pass
del content
try:
self.rsync_initial_timeout = \
int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
except ValueError:
self.rsync_initial_timeout = 15
try:
maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
except SystemExit as e:
raise # Needed else can't exit
except:
maxretries = -1 #default number of retries
if syncuri.startswith("file://"):
self.proto = "file"
dosyncuri = syncuri[7:]
unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
dosyncuri, timestamp, opts)
self._process_exitcode(exitcode, dosyncuri, out, 1)
if exitcode == 0:
if unchanged:
self.repo_storage.abort_update()
else:
self.repo_storage.commit_update()
self.repo_storage.garbage_collection()
return (exitcode, updatecache_flg)
retries=0
try:
self.proto, user_name, hostname, port = re.split(
r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
syncuri, maxsplit=4)[1:5]
except ValueError:
writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri,
noiselevel=-1, level=logging.ERROR)
return (1, False)
self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
if port is None:
port=""
if user_name is None:
user_name=""
if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
getaddrinfo_host = hostname
else:
# getaddrinfo needs the brackets stripped
getaddrinfo_host = hostname[1:-1]
updatecache_flg = False
all_rsync_opts = set(self.rsync_opts)
all_rsync_opts.update(self.extra_rsync_opts)
family = socket.AF_UNSPEC
if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
family = socket.AF_INET
elif socket.has_ipv6 and \
("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
family = socket.AF_INET6
addrinfos = None
uris = []
try:
addrinfos = getaddrinfo_validate(
socket.getaddrinfo(getaddrinfo_host, None,
family, socket.SOCK_STREAM))
except socket.error as e:
writemsg_level(
"!!! getaddrinfo failed for '%s': %s\n"
% (_unicode_decode(hostname), _unicode(e)),
noiselevel=-1, level=logging.ERROR)
if addrinfos:
AF_INET = socket.AF_INET
AF_INET6 = None
if socket.has_ipv6:
AF_INET6 = socket.AF_INET6
ips_v4 = []
ips_v6 = []
for addrinfo in addrinfos:
if addrinfo[0] == AF_INET:
ips_v4.append("%s" % addrinfo[4][0])
elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
# IPv6 addresses need to be enclosed in square brackets
ips_v6.append("[%s]" % addrinfo[4][0])
random.shuffle(ips_v4)
random.shuffle(ips_v6)
# Give priority to the address family that
# getaddrinfo() returned first.
if AF_INET6 is not None and addrinfos and \
addrinfos[0][0] == AF_INET6:
ips = ips_v6 + ips_v4
else:
ips = ips_v4 + ips_v6
for ip in ips:
uris.append(syncuri.replace(
"//" + user_name + hostname + port + "/",
"//" + user_name + ip + port + "/", 1))
if not uris:
# With some configurations we need to use the plain hostname
# rather than try to resolve the ip addresses (bug #340817).
uris.append(syncuri)
# reverse, for use with pop()
uris.reverse()
uris_orig = uris[:]
effective_maxretries = maxretries
if effective_maxretries < 0:
effective_maxretries = len(uris) - 1
local_state_unchanged = True
while (1):
if uris:
dosyncuri = uris.pop()
elif maxretries < 0 or retries > maxretries:
writemsg("!!! Exhausted addresses for %s\n"
% _unicode_decode(hostname), noiselevel=-1)
return (1, False)
else:
uris.extend(uris_orig)
dosyncuri = uris.pop()
if (retries==0):
if "--ask" in opts:
uq = UserQuery(opts)
if uq.query("Do you want to sync your ebuild repository " + \
"with the mirror at\n" + blue(dosyncuri) + bold("?"),
enter_invalid) == "No":
print()
print("Quitting.")
print()
sys.exit(128 + signal.SIGINT)
self.logger(self.xterm_titles,
">>> Starting rsync with " + dosyncuri)
if "--quiet" not in opts:
print(">>> Starting rsync with "+dosyncuri+"...")
else:
self.logger(self.xterm_titles,
">>> Starting retry %d of %d with %s" % \
(retries, effective_maxretries, dosyncuri))
writemsg_stdout(
"\n\n>>> Starting retry %d of %d with %s\n" % \
(retries, effective_maxretries, dosyncuri), noiselevel=-1)
if dosyncuri.startswith('ssh://'):
dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
dosyncuri, timestamp, opts)
if not unchanged:
local_state_unchanged = False
if is_synced:
break
retries=retries+1
if maxretries < 0 or retries <= maxretries:
print(">>> Retrying...")
else:
# over retries
# exit loop
exitcode = EXCEEDED_MAX_RETRIES
break
self._process_exitcode(exitcode, dosyncuri, out, maxretries)
if local_state_unchanged:
# The quarantine download_dir is not intended to exist
# in this case, so refer gemato to the normal repository
# location.
download_dir = self.repo.location
else:
download_dir = self.download_dir
# if synced successfully, verify now
if exitcode == 0 and self.verify_metamanifest:
if gemato is None:
writemsg_level("!!! Unable to verify: gemato-11.0+ is required\n",
level=logging.ERROR, noiselevel=-1)
exitcode = 127
else:
try:
# we always verify the Manifest signature, in case
# we had to deal with key revocation case
m = gemato.recursiveloader.ManifestRecursiveLoader(
os.path.join(download_dir, 'Manifest'),
verify_openpgp=True,
openpgp_env=openpgp_env,
max_jobs=self.verify_jobs)
if not m.openpgp_signed:
raise RuntimeError('OpenPGP signature not found on Manifest')
ts = m.find_timestamp()
if ts is None:
raise RuntimeError('Timestamp not found in Manifest')
if (self.max_age != 0 and
(datetime.datetime.utcnow() - ts.ts).days > self.max_age):
out.quiet = False
out.ewarn('Manifest is over %d days old, this is suspicious!' % (self.max_age,))
out.ewarn('You may want to try using another mirror and/or reporting this one:')
out.ewarn(' %s' % (dosyncuri,))
out.ewarn('')
out.quiet = quiet
out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
out.einfo('Valid OpenPGP signature found:')
out.einfo('- primary key: %s' % (
m.openpgp_signature.primary_key_fingerprint))
out.einfo('- subkey: %s' % (
m.openpgp_signature.fingerprint))
out.einfo('- timestamp: %s UTC' % (
m.openpgp_signature.timestamp))
# if nothing has changed, skip the actual Manifest
# verification
if not local_state_unchanged:
out.ebegin('Verifying %s' % (download_dir,))
m.assert_directory_verifies()
out.eend(0)
except GematoException as e:
writemsg_level("!!! Manifest verification failed:\n%s\n"
% (e,),
level=logging.ERROR, noiselevel=-1)
exitcode = 1
verify_failure = True
if exitcode == 0 and not local_state_unchanged:
self.repo_storage.commit_update()
self.repo_storage.garbage_collection()
return (exitcode, updatecache_flg)
finally:
# Don't delete the update if verification failed, in case
# the cause needs to be investigated.
if not verify_failure:
self.repo_storage.abort_update()
if openpgp_env is not None:
openpgp_env.close()
def _sync(self):
'''Internal sync function which performs only the sync'''
opts = self.options.get('emerge_config').opts
self.usersync_uid = self.options.get('usersync_uid', None)
enter_invalid = '--ask-enter-invalid' in opts
out = portage.output.EOutput()
syncuri = self.repo.sync_uri
vcs_dirs = frozenset(VCS_DIRS)
vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
for vcs_dir in vcs_dirs:
writemsg_level(("!!! %s appears to be under revision " + \
"control (contains %s).\n!!! Aborting rsync sync.\n") % \
(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
return (1, False)
self.timeout=180
rsync_opts = []
if self.settings["PORTAGE_RSYNC_OPTS"] == "":
rsync_opts = self._set_rsync_defaults()
else:
rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
self.extra_rsync_opts = portage.util.shlex_split(
self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
# Real local timestamp file.
self.servertimestampfile = os.path.join(
self.repo.location, "metadata", "timestamp.chk")
content = portage.util.grabfile(self.servertimestampfile)
timestamp = 0
if content:
try:
timestamp = time.mktime(time.strptime(content[0],
TIMESTAMP_FORMAT))
except (OverflowError, ValueError):
pass
del content
try:
self.rsync_initial_timeout = \
int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
except ValueError:
self.rsync_initial_timeout = 15
try:
maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
except SystemExit as e:
raise # Needed else can't exit
except:
maxretries = -1 #default number of retries
if syncuri.startswith("file://"):
self.proto = "file"
dosyncuri = syncuri[6:]
is_synced, exitcode = self._do_rsync(
dosyncuri, timestamp, opts)
self._process_exitcode(exitcode, dosyncuri, out, 1)
return (exitcode, exitcode == os.EX_OK)
retries=0
try:
self.proto, user_name, hostname, port = re.split(
r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
syncuri, maxsplit=4)[1:5]
except ValueError:
writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri,
noiselevel=-1, level=logging.ERROR)
return (1, False)
self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
if port is None:
port=""
if user_name is None:
user_name=""
if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
getaddrinfo_host = hostname
else:
# getaddrinfo needs the brackets stripped
getaddrinfo_host = hostname[1:-1]
updatecache_flg=True
all_rsync_opts = set(self.rsync_opts)
all_rsync_opts.update(self.extra_rsync_opts)
family = socket.AF_UNSPEC
if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
family = socket.AF_INET
elif socket.has_ipv6 and \
("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
family = socket.AF_INET6
addrinfos = None
uris = []
try:
addrinfos = getaddrinfo_validate(
socket.getaddrinfo(getaddrinfo_host, None,
family, socket.SOCK_STREAM))
except socket.error as e:
writemsg_level(
"!!! getaddrinfo failed for '%s': %s\n" % (hostname, e),
noiselevel=-1, level=logging.ERROR)
if addrinfos:
AF_INET = socket.AF_INET
AF_INET6 = None
if socket.has_ipv6:
AF_INET6 = socket.AF_INET6
ips_v4 = []
ips_v6 = []
for addrinfo in addrinfos:
if addrinfo[0] == AF_INET:
ips_v4.append("%s" % addrinfo[4][0])
elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
# IPv6 addresses need to be enclosed in square brackets
ips_v6.append("[%s]" % addrinfo[4][0])
random.shuffle(ips_v4)
random.shuffle(ips_v6)
# Give priority to the address family that
# getaddrinfo() returned first.
if AF_INET6 is not None and addrinfos and \
addrinfos[0][0] == AF_INET6:
ips = ips_v6 + ips_v4
else:
ips = ips_v4 + ips_v6
for ip in ips:
uris.append(syncuri.replace(
"//" + user_name + hostname + port + "/",
"//" + user_name + ip + port + "/", 1))
if not uris:
# With some configurations we need to use the plain hostname
# rather than try to resolve the ip addresses (bug #340817).
uris.append(syncuri)
# reverse, for use with pop()
uris.reverse()
effective_maxretries = maxretries
if effective_maxretries < 0:
effective_maxretries = len(uris) - 1
while (1):
if uris:
dosyncuri = uris.pop()
else:
writemsg("!!! Exhausted addresses for %s\n" % \
hostname, noiselevel=-1)
return (1, False)
if (retries==0):
if "--ask" in opts:
uq = UserQuery(opts)
if uq.query("Do you want to sync your Portage tree " + \
"with the mirror at\n" + blue(dosyncuri) + bold("?"),
enter_invalid) == "No":
print()
print("Quitting.")
print()
sys.exit(128 + signal.SIGINT)
self.logger(self.xterm_titles,
">>> Starting rsync with " + dosyncuri)
if "--quiet" not in opts:
print(">>> Starting rsync with "+dosyncuri+"...")
else:
self.logger(self.xterm_titles,
">>> Starting retry %d of %d with %s" % \
(retries, effective_maxretries, dosyncuri))
writemsg_stdout(
"\n\n>>> Starting retry %d of %d with %s\n" % \
(retries, effective_maxretries, dosyncuri), noiselevel=-1)
if dosyncuri.startswith('ssh://'):
dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
if is_synced:
break
retries=retries+1
if maxretries < 0 or retries <= maxretries:
print(">>> Retrying...")
else:
# over retries
# exit loop
updatecache_flg=False
exitcode = EXCEEDED_MAX_RETRIES
break
self._process_exitcode(exitcode, dosyncuri, out, maxretries)
return (exitcode, updatecache_flg)
def main(settings=None, logger=None):
"""Main program operation method....
@param settings: dict. defaults to settings.DEFAULTS
@param logger: python logging module defaults to init_logger(settings)
@return boolean success/failure
"""
if settings is None:
print("NO Input settings, using defaults...")
settings = DEFAULTS.copy()
if logger is None:
logger = init_logger(settings)
_libs_to_check = settings["library"]
if not settings["stdout"].isatty() or settings["nocolor"]:
nocolor()
logger.warning(
blue(" * ") + yellow("This is the new python coded version"))
logger.warning(
blue(" * ") + yellow("Please report any bugs found using it."))
logger.warning(
blue(" * ") + yellow("The original revdep-rebuild script is "
"installed as revdep-rebuild.sh"))
logger.warning(
blue(" * ") + yellow("Please file bugs at: "
"https://bugs.gentoo.org/"))
if os.getuid() != 0 and not settings["PRETEND"]:
logger.warning(
blue(" * ") +
yellow("You are not root, adding --pretend to portage options"))
settings["PRETEND"] = True
logger.debug("\tmain(), _libs_to_check = %s" % str(_libs_to_check))
if settings["USE_TMP_FILES"] and check_temp_files(
settings["DEFAULT_TMP_DIR"], logger=logger):
libraries, la_libraries, libraries_links, binaries = read_cache(
settings["DEFAULT_TMP_DIR"])
assigned, orphaned = analyse(
settings=settings,
logger=logger,
libraries=libraries,
la_libraries=la_libraries,
libraries_links=libraries_links,
binaries=binaries,
_libs_to_check=_libs_to_check,
)
else:
assigned, orphaned = analyse(settings,
logger,
_libs_to_check=_libs_to_check)
if not assigned and not orphaned:
logger.warning("\n" + bold("Your system is consistent"))
# return the correct exit code
return 0
elif orphaned:
# blank line for beter visibility of the following lines
logger.warning("")
if settings["library"]:
logger.warning(
red(" !!! Dependant orphaned files: ") +
bold("No installed package was found for the following:"))
else:
logger.warning(
red(" !!! Broken orphaned files: ") +
bold("No installed package was found for the following:"))
for filename in orphaned:
logger.warning(red("\t* ") + filename)
success = rebuild(logger, assigned, settings)
logger.debug("rebuild return code = %i" % success)
return success
def _create_use_string(
conf, name, cur_iuse, iuse_forced, cur_use, old_iuse, old_use, is_new, feature_flags, reinst_flags
):
if not conf.print_use_string:
return ""
enabled = []
if conf.alphabetical:
disabled = enabled
removed = enabled
else:
disabled = []
removed = []
cur_iuse = set(cur_iuse)
enabled_flags = cur_iuse.intersection(cur_use)
removed_iuse = set(old_iuse).difference(cur_iuse)
any_iuse = cur_iuse.union(old_iuse)
any_iuse = list(any_iuse)
any_iuse.sort()
for flag in any_iuse:
flag_str = None
isEnabled = False
reinst_flag = reinst_flags and flag in reinst_flags
if flag in enabled_flags:
isEnabled = True
if is_new or flag in old_use and (conf.all_flags or reinst_flag):
flag_str = red(flag)
elif flag not in old_iuse:
flag_str = yellow(flag) + "%*"
elif flag not in old_use:
flag_str = green(flag) + "*"
elif flag in removed_iuse:
if conf.all_flags or reinst_flag:
flag_str = yellow("-" + flag) + "%"
if flag in old_use:
flag_str += "*"
flag_str = "(" + flag_str + ")"
removed.append(flag_str)
continue
else:
if is_new or flag in old_iuse and flag not in old_use and (conf.all_flags or reinst_flag):
flag_str = blue("-" + flag)
elif flag not in old_iuse:
flag_str = yellow("-" + flag)
if flag not in iuse_forced:
flag_str += "%"
elif flag in old_use:
flag_str = green("-" + flag) + "*"
if flag_str:
if flag in feature_flags:
flag_str = "{" + flag_str + "}"
elif flag in iuse_forced:
flag_str = "(" + flag_str + ")"
if isEnabled:
enabled.append(flag_str)
else:
disabled.append(flag_str)
if conf.alphabetical:
ret = " ".join(enabled)
else:
ret = " ".join(enabled + disabled + removed)
if ret:
ret = '%s="%s" ' % (name, ret)
return ret
def collect_libraries_from_dir(dirs, mask, logger):
''' Collects all libraries from specified list of directories.
mask is list of pathes, that are ommited in scanning, can be eighter single file or entire directory
Returns tuple composed of: list of libraries, list of symlinks, and toupe with pair
(symlink_id, library_id) for resolving dependencies
'''
# contains list of directories found
# allows us to reduce number of fnc calls
found_directories = set()
found_files = set()
found_symlinks = set()
found_la_files = set() # la libraries
for _dir in dirs:
if _dir in mask:
continue
try:
for _listing in os.listdir(_dir):
listing = os.path.join(_dir, _listing)
if listing in mask or _listing in mask:
continue
if os.path.isdir(listing):
if os.path.islink(listing):
#we do not want scan symlink-directories
pass
else:
found_directories.add(listing)
elif os.path.isfile(listing):
if (listing.endswith('.so') or listing.endswith('.a')
or '.so.' in listing):
if os.path.islink(listing):
found_symlinks.add(listing)
else:
found_files.add(listing)
continue
elif listing.endswith('.la'):
if listing in found_la_files:
continue
found_la_files.add(listing)
else:
# sometimes there are binaries in libs' subdir,
# for example in nagios
if not os.path.islink(listing):
#if listing in found_files or listing in found_symlinks:
#continue
prv = os.stat(listing)[stat.ST_MODE]
if prv & stat.S_IXUSR == stat.S_IXUSR or \
prv & stat.S_IXGRP == stat.S_IXGRP or \
prv & stat.S_IXOTH == stat.S_IXOTH:
found_files.add(listing)
except Exception as ex:
logger.debug('\t' + yellow('Exception collecting libraries: ' +
blue('%s') % str(ex)))
if found_directories:
_file, la_file, link = \
collect_libraries_from_dir(found_directories, mask, logger)
found_files.update(_file)
found_la_files.update(la_file)
found_symlinks.update(link)
return (found_files, found_la_files, found_symlinks)
def swig_generate(self):
"build all available modules"
quote = lambda s: '"' + s + '"'
for module in self.MODULES:
module_name = self.MODULES[module][0]
config_cmd = self.MODULES[module][1]
module_pkg_name = self.MODULES[module][2]
mod_hack_name = self.MODULES[module][3]
mod_out_prefix = module_pkg_name.replace('.', os.sep) + module
if sys.platform == "win32":
INCLUDE_DIR = os.path.join(os.getenv("COINDIR"), "include")
CPP_FLAGS = "-I" + quote(INCLUDE_DIR) + " " + \
"-I" + quote(os.path.join(os.getenv("COINDIR"), "include", "Inventor", "annex")) + \
" /DCOIN_DLL /wd4244 /wd4049"
# aquire highest non-debug Coin library version
try:
LDFLAGS_LIBS = quote(
max(
glob.glob(
os.path.join(os.getenv("COINDIR"), "lib",
"coin?.lib")))) + " "
# with cmake the coin library is named Coin4.lib
except ValueError:
LDFLAGS_LIBS = quote(
max(
glob.glob(
os.path.join(os.getenv("COINDIR"), "lib",
"Coin?.lib")))) + " "
if module == "sowin":
CPP_FLAGS += " /DSOWIN_DLL"
LDFLAGS_LIBS += quote(
os.path.join(os.getenv("COINDIR"), "lib",
"sowin1.lib"))
elif module == "soqt":
CPP_FLAGS += " -I" + '"' + os.getenv(
"QTDIR") + "\\include\" /DSOQT_DLL"
if os.path.isdir(os.getenv("QTDIR") + "\\include\Qt\""):
CPP_FLAGS += " -I" + '"' + os.getenv(
"QTDIR") + "\\include\Qt\""
LDFLAGS_LIBS += os.path.join(os.getenv("COINDIR"),
"lib", "soqt1.lib") + " "
else:
# workaround for conda qt4:
CPP_FLAGS += " -I" + '"' + os.getenv(
"QTDIR") + "\\include\qt\Qt\""
CPP_FLAGS += " -I" + '"' + os.getenv(
"QTDIR") + "\\include\qt\""
LDFLAGS_LIBS += os.path.join(os.getenv("COINDIR"),
"lib", "SoQt.lib") + " "
else:
INCLUDE_DIR = self.do_os_popen("coin-config --includedir")
if module_name != 'coin':
mod_include_dir = self.do_os_popen("%s --includedir" %
config_cmd)
if mod_include_dir != INCLUDE_DIR:
INCLUDE_DIR += '\" -I\"%s' % mod_include_dir
CPP_FLAGS = self.do_os_popen(
"%s --cppflags" %
config_cmd) + " -Wno-unused -Wno-maybe-uninitialized"
LDFLAGS_LIBS = self.do_os_popen("%s --ldflags --libs" %
config_cmd)
if not os.path.isfile(mod_out_prefix + "_wrap.cpp"):
print(
red("\n=== Generating %s_wrap.cpp for %s ===\n" %
(mod_out_prefix, module)))
print(
blue(self.SWIG + " " + self.SWIG_SUPPRESS_WARNINGS + " " +
self.SWIG_PARAMS %
(INCLUDE_DIR, self.CXX_INCS, mod_out_prefix, module)))
if os.system(self.SWIG + " " + self.SWIG_SUPPRESS_WARNINGS +
" " + self.SWIG_PARAMS %
(INCLUDE_DIR, self.CXX_INCS, mod_out_prefix,
mod_hack_name)):
print(
red("SWIG did not generate wrappers successfully! ** Aborting **"
))
sys.exit(1)
else:
print(
red("=== %s_wrap.cpp for %s already exists! ===" %
(mod_out_prefix, module_pkg_name + module)))
self.ext_modules.append(
Extension(
module_name, [mod_out_prefix + "_wrap.cpp"],
extra_compile_args=(self.CXX_INCS + CPP_FLAGS).split(),
extra_link_args=(self.CXX_LIBS + LDFLAGS_LIBS).split()))
def analyse(settings,
logger,
libraries=None,
la_libraries=None,
libraries_links=None,
binaries=None,
_libs_to_check=None):
"""Main program body. It will collect all info and determine the
pkgs needing rebuilding.
@param logger: logger used for logging messages, instance of logging.Logger
class. Can be logging (RootLogger).
@param _libs_to_check Libraries that need to be checked only
@rtype list: list of pkgs that need rebuilding
"""
searchbits = set()
'''if _libs_to_check:
for lib in _libs_to_check:
if "lib64" in lib:
searchbits.add('64')
elif "lib32" in lib:
searchbits.add('32')
else:
_libs_to_check = set()'''
searchbits.update(['64', '32'])
masked_dirs, masked_files, ld = parse_revdep_config(
settings['REVDEP_CONFDIR'])
masked_dirs.update([
'/lib/modules',
'/lib32/modules',
'/lib64/modules',
])
if '64' not in searchbits:
masked_dirs.update(['/lib64', '/usr/lib64'])
elif '32' not in searchbits:
masked_dirs.update(['/lib32', '/usr/lib32'])
all_masks = masked_dirs.copy()
all_masks.update(masked_files)
logger.debug("\tall_masks:")
for x in sorted(all_masks):
logger.debug('\t\t%s' % (x))
if libraries and la_libraries and libraries_links and binaries:
logger.info(
blue(' * ') +
bold('Found a valid cache, skipping collecting phase'))
else:
#TODO: add partial cache (for ex. only libraries)
# when found for some reason
stime = current_milli_time()
logger.warning(
green(' * ') + bold('Collecting system binaries and libraries'))
bin_dirs, lib_dirs = prepare_search_dirs(logger, settings)
lib_dirs.update(ld)
bin_dirs.update(ld)
logger.debug('\tanalyse(), bin directories:')
for x in sorted(bin_dirs):
logger.debug('\t\t%s' % (x))
logger.debug('\tanalyse(), lib directories:')
for x in sorted(lib_dirs):
logger.debug('\t\t%s' % (x))
logger.debug('\tanalyse(), masked directories:')
for x in sorted(masked_dirs):
logger.debug('\t\t%s' % (x))
logger.debug('\tanalyse(), masked files:')
for x in sorted(masked_files):
logger.debug('\t\t%s' % (x))
ftime = current_milli_time()
logger.debug('\ttime to complete task: %d milliseconds' %
(ftime - stime))
stime = current_milli_time()
logger.info(
green(' * ') + bold('Collecting dynamic linking informations'))
libraries, la_libraries, libraries_links = \
collect_libraries_from_dir(lib_dirs, all_masks, logger)
binaries = collect_binaries_from_dir(bin_dirs, all_masks, logger)
ftime = current_milli_time()
logger.debug('\ttime to complete task: %d milliseconds' %
(ftime - stime))
if settings['USE_TMP_FILES']:
save_cache(logger=logger,
to_save={
'libraries': libraries,
'la_libraries': la_libraries,
'libraries_links': libraries_links,
'binaries': binaries
},
temp_path=settings['DEFAULT_TMP_DIR'])
logger.debug(
'\tanalyse(), Found %i libraries (+%i symlinks) and %i binaries' %
(len(libraries), len(libraries_links), len(binaries)))
logger.info(green(' * ') + bold('Scanning files'))
libs_and_bins = libraries.union(binaries)
scanned_files = scan_files(libs_and_bins, settings['CMD_MAX_ARGS'], logger,
searchbits)
logger.warning(green(' * ') + bold('Checking dynamic linking consistency'))
logger.debug(
'\tanalyse(), Searching for %i libs, bins within %i libraries and links'
% (len(libs_and_bins), len(libraries) + len(libraries_links)))
libcheck = LibCheck(scanned_files, logger, _libs_to_check, searchbits,
all_masks, masked_dirs)
broken_pathes = libcheck.process_results(libcheck.search())
broken_la = extract_dependencies_from_la(la_libraries,
libraries.union(libraries_links),
_libs_to_check, logger)
broken_pathes += broken_la
if broken_pathes:
logger.warning(green(' * ') + bold('Assign files to packages'))
return assign_packages(broken_pathes, logger, settings)
return None, None # no need to assign anything
