def unlock(self):
"""
Set GPG_TTY and run GPG unlock command.
If gpg-keepalive is set, start keepalive thread.
"""
if self.GPG_unlock_command and (
self.settings.get("BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0])
== "gpkg"
):
try:
os.environ["GPG_TTY"] = os.ttyname(sys.stdout.fileno())
except OSError as e:
# When run with no input/output tty, this will fail.
# However, if the password is given by command,
# GPG does not need to ask password, so can be ignored.
writemsg(colorize("WARN", str(e)) + "\n")
cmd = shlex_split(varexpand(self.GPG_unlock_command, mydict=self.settings))
return_code = subprocess.Popen(cmd).wait()
if return_code == os.EX_OK:
writemsg_stdout(colorize("GOOD", "unlocked") + "\n")
sys.stdout.flush()
else:
raise GPGException("GPG unlock failed")
if self.keepalive:
self.GPG_unlock_command = shlex_split(
varexpand(self.GPG_unlock_command, mydict=self.settings)
)
self.thread = threading.Thread(target=self.gpg_keepalive, daemon=True)
self.thread.start()
def new(self, **kwargs):
'''Do the initial clone of the repository'''
if kwargs:
self._kwargs(kwargs)
if not self.has_bin:
return (1, False)
try:
if not os.path.exists(self.repo.location):
os.makedirs(self.repo.location)
self.logger(self.xterm_titles,
'Created new directory %s' % self.repo.location)
except IOError:
return (1, False)
sync_uri = self.repo.sync_uri
if sync_uri.startswith("file://"):
sync_uri = sync_uri[7:]
git_cmd_opts = ""
if self.repo.module_specific_options.get('sync-git-env'):
shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-env'])
env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(env)
if self.repo.module_specific_options.get('sync-git-clone-env'):
shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-clone-env'])
clone_env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(clone_env)
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
if self.repo.clone_depth is not None:
if self.repo.clone_depth != 0:
git_cmd_opts += " --depth %d" % self.repo.clone_depth
elif self.repo.sync_depth is not None:
if self.repo.sync_depth != 0:
git_cmd_opts += " --depth %d" % self.repo.sync_depth
else:
# default
git_cmd_opts += " --depth 1"
if self.repo.module_specific_options.get('sync-git-clone-extra-opts'):
git_cmd_opts += " %s" % self.repo.module_specific_options['sync-git-clone-extra-opts']
git_cmd = "%s clone%s %s ." % (self.bin_command, git_cmd_opts,
portage._shell_quote(sync_uri))
writemsg_level(git_cmd + "\n")
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), git_cmd),
**self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! git clone error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
if not self.verify_head():
return (1, False)
return (os.EX_OK, True)
def new(self, **kwargs):
'''Do the initial clone of the repository'''
if kwargs:
self._kwargs(kwargs)
if not self.has_bin:
return (1, False)
try:
if not os.path.exists(self.repo.location):
os.makedirs(self.repo.location)
self.logger(self.xterm_titles,
'Created new directory %s' % self.repo.location)
except IOError:
return (1, False)
sync_uri = self.repo.sync_uri
if sync_uri.startswith("file://"):
sync_uri = sync_uri[7:]
git_cmd_opts = ""
if self.repo.module_specific_options.get('sync-git-env'):
shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-env'])
env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(env)
if self.repo.module_specific_options.get('sync-git-clone-env'):
shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-clone-env'])
clone_env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(clone_env)
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
if self.repo.clone_depth is not None:
if self.repo.clone_depth != 0:
git_cmd_opts += " --depth %d" % self.repo.clone_depth
elif self.repo.sync_depth is not None:
if self.repo.sync_depth != 0:
git_cmd_opts += " --depth %d" % self.repo.sync_depth
else:
# default
git_cmd_opts += " --depth 1"
if self.repo.module_specific_options.get('sync-git-clone-extra-opts'):
git_cmd_opts += " %s" % self.repo.module_specific_options['sync-git-clone-extra-opts']
git_cmd = "%s clone%s %s ." % (self.bin_command, git_cmd_opts,
portage._shell_quote(sync_uri))
writemsg_level(git_cmd + "\n")
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), git_cmd),
**self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! git clone error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
if not self.verify_head():
return (1, False)
return (os.EX_OK, True)
def new(self, **kwargs):
"""Do the initial clone of the repository"""
if kwargs:
self._kwargs(kwargs)
try:
if not os.path.exists(self.repo.location):
os.makedirs(self.repo.location)
self.logger(self.xterm_titles,
"Created new directory %s" % self.repo.location)
except IOError:
return (1, False)
sync_uri = self.repo.sync_uri
if sync_uri.startswith("file://"):
sync_uri = sync_uri[7:]
hg_cmd_opts = ""
if self.repo.module_specific_options.get("sync-mercurial-env"):
shlexed_env = shlex_split(
self.repo.module_specific_options["sync-mercurial-env"])
env = dict(
(k, v) for k, _, v in (assignment.partition("=")
for assignment in shlexed_env) if k)
self.spawn_kwargs["env"].update(env)
if self.repo.module_specific_options.get("sync-mercurial-clone-env"):
shlexed_env = shlex_split(
self.repo.module_specific_options["sync-mercurial-clone-env"])
clone_env = dict(
(k, v) for k, _, v in (assignment.partition("=")
for assignment in shlexed_env) if k)
self.spawn_kwargs["env"].update(clone_env)
if self.settings.get("PORTAGE_QUIET") == "1":
hg_cmd_opts += " --quiet"
if self.repo.module_specific_options.get(
"sync-mercurial-clone-extra-opts"):
hg_cmd_opts += (
" %s" % self.repo.
module_specific_options["sync-mercurial-clone-extra-opts"])
hg_cmd = "%s clone%s %s ." % (
self.bin_command,
hg_cmd_opts,
portage._shell_quote(sync_uri),
)
writemsg_level(hg_cmd + "\n")
exitcode = portage.process.spawn(shlex_split(hg_cmd),
cwd=portage._unicode_encode(
self.repo.location),
**self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! hg clone error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
return (os.EX_OK, True)
def update(self):
''' Update existing git repository, and ignore the syncuri. We are
going to trust the user and assume that the user is in the branch
that he/she wants updated. We'll let the user manage branches with
git directly.
'''
if not self.has_bin:
return (1, False)
git_cmd_opts = ""
if self.repo.module_specific_options.get('sync-git-env'):
shlexed_env = shlex_split(
self.repo.module_specific_options['sync-git-env'])
env = dict(
(k, v) for k, _, v in (assignment.partition('=')
for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(env)
if self.repo.module_specific_options.get('sync-git-pull-env'):
shlexed_env = shlex_split(
self.repo.module_specific_options['sync-git-pull-env'])
pull_env = dict(
(k, v) for k, _, v in (assignment.partition('=')
for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(pull_env)
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
if self.repo.module_specific_options.get('sync-git-pull-extra-opts'):
git_cmd_opts += " %s" % self.repo.module_specific_options[
'sync-git-pull-extra-opts']
git_cmd = "%s pull%s" % (self.bin_command, git_cmd_opts)
writemsg_level(git_cmd + "\n")
rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
previous_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(
self.repo.location))
exitcode = portage.process.spawn_bash(
"cd %s ; exec %s" %
(portage._shell_quote(self.repo.location), git_cmd),
**self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! git pull error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
if not self.verify_head():
return (1, False)
current_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(
self.repo.location))
return (os.EX_OK, current_rev != previous_rev)
def update(self):
"""Update existing mercurial repository, and ignore the syncuri. We are
going to trust the user and assume that the user is in the branch
that he/she wants updated. We'll let the user manage branches with
hg directly.
"""
hg_cmd_opts = ""
if self.repo.module_specific_options.get("sync-mercurial-env"):
shlexed_env = shlex_split(
self.repo.module_specific_options["sync-mercurial-env"])
env = dict(
(k, v) for k, _, v in (assignment.partition("=")
for assignment in shlexed_env) if k)
self.spawn_kwargs["env"].update(env)
if self.repo.module_specific_options.get("sync-mercurial-pull-env"):
shlexed_env = shlex_split(
self.repo.module_specific_options["sync-mercurial-pull-env"])
pull_env = dict(
(k, v) for k, _, v in (assignment.partition("=")
for assignment in shlexed_env) if k)
self.spawn_kwargs["env"].update(pull_env)
if self.settings.get("PORTAGE_QUIET") == "1":
hg_cmd_opts += " --quiet"
if self.repo.module_specific_options.get(
"sync-mercurial-pull-extra-opts"):
hg_cmd_opts += (
" %s" % self.repo.
module_specific_options["sync-mercurial-pull-extra-opts"])
hg_cmd = "%s pull -u%s" % (self.bin_command, hg_cmd_opts)
writemsg_level(hg_cmd + "\n")
rev_cmd = [self.bin_command, "id", "--id", "--rev", "tip"]
previous_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(
self.repo.location))
exitcode = portage.process.spawn(shlex_split(hg_cmd),
cwd=portage._unicode_encode(
self.repo.location),
**self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! hg pull error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
current_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(
self.repo.location))
return (os.EX_OK, current_rev != previous_rev)
def set_port_dirs(self, portdir, portdir_overlay): self.portdir = portdir self.portdir_overlay = portdir_overlay if self.portdir_overlay is None: self.portdir_overlay = "" self.overlay_profiles = [] for ov in shlex_split(self.portdir_overlay): ov = normalize_path(ov) profiles_dir = os.path.join(ov, "profiles") if os.path.isdir(profiles_dir): self.overlay_profiles.append(profiles_dir) self.profile_locations = [os.path.join(portdir, "profiles")] + self.overlay_profiles self.profile_and_user_locations = self.profile_locations[:] if self._user_config: self.profile_and_user_locations.append(self.abs_user_config) self.profile_locations = tuple(self.profile_locations) self.profile_and_user_locations = tuple(self.profile_and_user_locations) self.pmask_locations = [os.path.join(portdir, "profiles")] self.pmask_locations.extend(self.profiles) self.pmask_locations.extend(self.overlay_profiles) self.pmask_locations = tuple(self.pmask_locations)
def diffstatusoutput(cmd, file1, file2): """ Execute the string cmd in a shell with getstatusoutput() and return a 2-tuple (status, output). """ # Use Popen to emulate getstatusoutput(), since getstatusoutput() may # raise a UnicodeDecodeError which makes the output inaccessible. args = shlex_split(cmd % (file1, file2)) if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ not os.path.isabs(args[0]): # Python 3.1 _execvp throws TypeError for non-absolute executable # path passed as bytes (see http://bugs.python.org/issue8513). fullname = portage.process.find_binary(args[0]) if fullname is None: raise portage.exception.CommandNotFound(args[0]) args[0] = fullname args = [portage._unicode_encode(x, errors='strict') for x in args] proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = portage._unicode_decode(proc.communicate()[0]) if output and output[-1] == "\n": # getstatusoutput strips one newline output = output[:-1] return (proc.wait(), output)
def diffstatusoutput(cmd, file1, file2):
"""
Execute the string cmd in a shell with getstatusoutput() and return a
2-tuple (status, output).
"""
# Use Popen to emulate getstatusoutput(), since getstatusoutput() may
# raise a UnicodeDecodeError which makes the output inaccessible.
args = shlex_split(cmd % (file1, file2))
if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
not os.path.isabs(args[0]):
# Python 3.1 _execvp throws TypeError for non-absolute executable
# path passed as bytes (see http://bugs.python.org/issue8513).
fullname = portage.process.find_binary(args[0])
if fullname is None:
raise portage.exception.CommandNotFound(args[0])
args[0] = fullname
args = [portage._unicode_encode(x, errors='strict') for x in args]
proc = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = portage._unicode_decode(proc.communicate()[0])
if output and output[-1] == "\n":
# getstatusoutput strips one newline
output = output[:-1]
return (proc.wait(), output)
def updated_config_files(count):
self.assertEqual(
count,
sum(
len(x[1]) for x in find_updated_config_files(
eroot, shlex_split(config_protect))),
)
def _exclude_pattern(s):
# shlex_split enables quoted whitespace inside patterns
if s:
pat = re.compile("|".join(
fnmatch.translate(x.lstrip(os.sep)) for x in shlex_split(s)))
else:
pat = None
return pat
def update(self):
''' Update existing git repository, and ignore the syncuri. We are
going to trust the user and assume that the user is in the branch
that he/she wants updated. We'll let the user manage branches with
git directly.
'''
if not self.has_bin:
return (1, False)
git_cmd_opts = ""
if self.repo.module_specific_options.get('sync-git-env'):
shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-env'])
env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(env)
if self.repo.module_specific_options.get('sync-git-pull-env'):
shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-pull-env'])
pull_env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(pull_env)
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
if self.repo.module_specific_options.get('sync-git-pull-extra-opts'):
git_cmd_opts += " %s" % self.repo.module_specific_options['sync-git-pull-extra-opts']
git_cmd = "%s pull%s" % (self.bin_command, git_cmd_opts)
writemsg_level(git_cmd + "\n")
rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
previous_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(self.repo.location))
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), git_cmd),
**self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! git pull error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
if not self.verify_head():
return (1, False)
current_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(self.repo.location))
return (os.EX_OK, current_rev != previous_rev)
def _exclude_pattern(s):
# shlex_split enables quoted whitespace inside patterns
if s:
pat = re.compile("|".join(
fnmatch.translate(x.lstrip(os.sep))
for x in shlex_split(s)))
else:
pat = None
return pat
def _async_start(self):
pipe_logger = None
filter_proc = None
try:
log_input = None
if self.log_path is not None:
log_filter_file = self.log_filter_file
if log_filter_file is not None:
split_value = shlex_split(log_filter_file)
log_filter_file = split_value if split_value else None
if log_filter_file:
filter_input, stdin = os.pipe()
log_input, filter_output = os.pipe()
try:
filter_proc = yield asyncio.create_subprocess_exec(
*log_filter_file,
env=self.env,
stdin=filter_input,
stdout=filter_output,
stderr=filter_output,
loop=self.scheduler)
except EnvironmentError:
# Maybe the command is missing or broken somehow...
os.close(filter_input)
os.close(stdin)
os.close(log_input)
os.close(filter_output)
else:
self._stdin = os.fdopen(stdin, 'wb', 0)
os.close(filter_input)
os.close(filter_output)
if self._stdin is None:
# Since log_filter_file is unspecified or refers to a file
# that is missing or broken somehow, create a pipe that
# logs directly to pipe_logger.
log_input, stdin = os.pipe()
self._stdin = os.fdopen(stdin, 'wb', 0)
# Set background=True so that pipe_logger does not log to stdout.
pipe_logger = PipeLogger(background=True,
scheduler=self.scheduler,
input_fd=log_input,
log_file_path=self.log_path)
yield pipe_logger.async_start()
except asyncio.CancelledError:
if pipe_logger is not None and pipe_logger.poll() is None:
pipe_logger.cancel()
if filter_proc is not None and filter_proc.returncode is None:
filter_proc.terminate()
raise
self._main_task = asyncio.ensure_future(self._main(
pipe_logger, filter_proc=filter_proc),
loop=self.scheduler)
self._main_task.add_done_callback(self._main_exit)
def _start(self):
filter_proc = None
log_input = None
if self.log_path is not None:
log_filter_file = self.log_filter_file
if log_filter_file is not None:
split_value = shlex_split(log_filter_file)
log_filter_file = split_value if split_value else None
if log_filter_file:
filter_input, stdin = os.pipe()
log_input, filter_output = os.pipe()
try:
filter_proc = PopenProcess(
proc=subprocess.Popen(
log_filter_file,
env=self.env,
stdin=filter_input,
stdout=filter_output,
stderr=filter_output,
),
scheduler=self.scheduler,
)
filter_proc.start()
except EnvironmentError:
# Maybe the command is missing or broken somehow...
os.close(filter_input)
os.close(stdin)
os.close(log_input)
os.close(filter_output)
else:
self._stdin = os.fdopen(stdin, "wb", 0)
os.close(filter_input)
os.close(filter_output)
if self._stdin is None:
# Since log_filter_file is unspecified or refers to a file
# that is missing or broken somehow, create a pipe that
# logs directly to pipe_logger.
log_input, stdin = os.pipe()
self._stdin = os.fdopen(stdin, "wb", 0)
# Set background=True so that pipe_logger does not log to stdout.
pipe_logger = PipeLogger(
background=True,
scheduler=self.scheduler,
input_fd=log_input,
log_file_path=self.log_path,
)
pipe_logger.start()
self._main_task_cancel = functools.partial(self._main_cancel,
filter_proc, pipe_logger)
self._main_task = asyncio.ensure_future(self._main(
filter_proc, pipe_logger),
loop=self.scheduler)
self._main_task.add_done_callback(self._main_exit)
def _start_gpg_proc(self):
gpg_vars = self.gpg_vars
if gpg_vars is None:
gpg_vars = {}
else:
gpg_vars = gpg_vars.copy()
gpg_vars["FILE"] = self._manifest_path
gpg_cmd = varexpand(self.gpg_cmd, mydict=gpg_vars)
gpg_cmd = shlex_split(gpg_cmd)
gpg_proc = PopenProcess(proc=subprocess.Popen(gpg_cmd))
self._start_task(gpg_proc, self._gpg_proc_exit)
def _start_gpg_proc(self):
gpg_vars = self.gpg_vars
if gpg_vars is None:
gpg_vars = {}
else:
gpg_vars = gpg_vars.copy()
gpg_vars["FILE"] = self._manifest_path
gpg_cmd = varexpand(self.gpg_cmd, mydict=gpg_vars)
gpg_cmd = shlex_split(gpg_cmd)
gpg_proc = PopenProcess(proc=subprocess.Popen(gpg_cmd))
self._start_task(gpg_proc, self._gpg_proc_exit)
def validate_cmd_var(v): """ Validate an evironment variable value to see if it contains an executable command as the first token. returns (valid, token_list) where 'valid' is boolean and 'token_list' is the (possibly empty) list of tokens split by shlex. """ invalid = False v_split = shlex_split(v) if not v_split: invalid = True elif os.path.isabs(v_split[0]): invalid = not os.access(v_split[0], os.EX_OK) elif find_binary(v_split[0]) is None: invalid = True return (not invalid, v_split)
def _start_gpg_proc(self):
gpg_vars = self.gpg_vars
if gpg_vars is None:
gpg_vars = {}
else:
gpg_vars = gpg_vars.copy()
gpg_vars["FILE"] = self._manifest_path
gpg_cmd = varexpand(self.gpg_cmd, mydict=gpg_vars)
gpg_cmd = shlex_split(gpg_cmd)
gpg_proc = PopenProcess(proc=subprocess.Popen(gpg_cmd,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT))
# PipeLogger echos output and efficiently monitors for process
# exit by listening for the stdout EOF event.
gpg_proc.pipe_reader = PipeLogger(background=self.background,
input_fd=gpg_proc.proc.stdout, scheduler=self.scheduler)
self._start_task(gpg_proc, self._gpg_proc_exit)
def validate_cmd_var(v):
"""
Validate an evironment variable value to see if it
contains an executable command as the first token.
returns (valid, token_list) where 'valid' is boolean and 'token_list'
is the (possibly empty) list of tokens split by shlex.
"""
invalid = False
v_split = shlex_split(v)
if not v_split:
invalid = True
elif os.path.isabs(v_split[0]):
invalid = not os.access(v_split[0], os.EX_OK)
elif find_binary(v_split[0]) is None:
invalid = True
return (not invalid, v_split)
def _start_gpg_proc(self):
gpg_vars = self.gpg_vars
if gpg_vars is None:
gpg_vars = {}
else:
gpg_vars = gpg_vars.copy()
gpg_vars["FILE"] = self._manifest_path
gpg_cmd = varexpand(self.gpg_cmd, mydict=gpg_vars)
gpg_cmd = shlex_split(gpg_cmd)
gpg_proc = PopenProcess(proc=subprocess.Popen(gpg_cmd,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT))
# PipeLogger echos output and efficiently monitors for process
# exit by listening for the stdout EOF event.
gpg_proc.pipe_reader = PipeLogger(background=self.background,
input_fd=gpg_proc.proc.stdout, scheduler=self.scheduler)
self._start_task(gpg_proc, self._gpg_proc_exit)
def clean(self, **kwargs):
"""Log directory cleaning function
@param **kwargs: optional dictionary of values used in this function are:
settings: portage settings instance: defaults to portage.settings
"PORT_LOGDIR": directory to clean
"PORT_LOGDIR_CLEAN": command for cleaning the logs.
options: dict:
'NUM': int: number of days
'pretend': boolean
"""
num_of_days = None
pretend = False
if kwargs:
# convuluted, I know, but portage.settings does not exist in
# kwargs.get() when called from _emerge.main.clean_logs()
settings = kwargs.get('settings', None)
if not settings:
settings = portage.settings
options = kwargs.get('options', None)
if options:
num_of_days = options.get('NUM', None)
pretend = options.get('pretend', False)
clean_cmd = settings.get("PORT_LOGDIR_CLEAN")
if clean_cmd:
clean_cmd = shlex_split(clean_cmd)
if '-mtime' in clean_cmd and num_of_days is not None:
if num_of_days == 0:
i = clean_cmd.index('-mtime')
clean_cmd.remove('-mtime')
clean_cmd.pop(i)
else:
clean_cmd[clean_cmd.index('-mtime') +1] = \
'+%s' % str(num_of_days)
if pretend:
if "-delete" in clean_cmd:
clean_cmd.remove("-delete")
if not clean_cmd:
return (True, None)
rval = self._clean_logs(clean_cmd, settings)
errors = self._convert_errors(rval)
if errors:
return (False, errors)
return (True, None)
def diffstatusoutput(cmd, file1, file2): """ Execute the string cmd in a shell with getstatusoutput() and return a 2-tuple (status, output). """ # Use Popen to emulate getstatusoutput(), since getstatusoutput() may # raise a UnicodeDecodeError which makes the output inaccessible. args = shlex_split(cmd % (file1, file2)) args = [portage._unicode_encode(x, errors='strict') for x in args] proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = portage._unicode_decode(proc.communicate()[0]) if output and output[-1] == "\n": # getstatusoutput strips one newline output = output[:-1] return (proc.wait(), output)
def clean(self, **kwargs):
"""Log directory cleaning function
@param **kwargs: optional dictionary of values used in this function are:
settings: portage settings instance: defaults to portage.settings
"PORT_LOGDIR": directory to clean
"PORT_LOGDIR_CLEAN": command for cleaning the logs.
options: dict:
'NUM': int: number of days
'pretend': boolean
"""
num_of_days = None
pretend = False
# convoluted, I know, but portage.settings does not exist in
# kwargs.get() when called from _emerge.main.clean_logs()
settings = kwargs.get('settings', getattr(portage, 'settings', {}))
options = kwargs.get('options', None)
if options:
num_of_days = options.get('NUM', None)
pretend = options.get('pretend', False)
clean_cmd = settings.get("PORT_LOGDIR_CLEAN")
if clean_cmd:
clean_cmd = shlex_split(clean_cmd)
if '-mtime' in clean_cmd and num_of_days is not None:
if num_of_days == 0:
i = clean_cmd.index('-mtime')
clean_cmd.remove('-mtime')
clean_cmd.pop(i)
else:
clean_cmd[clean_cmd.index('-mtime') +1] = \
'+%s' % str(num_of_days)
if pretend:
if "-delete" in clean_cmd:
clean_cmd.remove("-delete")
if not clean_cmd:
return (True, None)
rval = self._clean_logs(clean_cmd, settings)
errors = self._convert_errors(rval)
if errors:
return (False, errors)
return (True, None)
def editor_is_executable(editor): """ Given an EDITOR string, validate that it refers to an executable. This uses shlex_split() to split the first component and do a PATH lookup if necessary. @param editor: An EDITOR value from the environment. @type: string @rtype: bool @return: True if an executable is found, False otherwise. """ editor_split = util.shlex_split(editor) if not editor_split: return False filename = editor_split[0] if not os.path.isabs(filename): return find_binary(filename) is not None return os.access(filename, os.X_OK) and os.path.isfile(filename)
def editor_is_executable(editor): """ Given an EDITOR string, validate that it refers to an executable. This uses shlex_split() to split the first component and do a PATH lookup if necessary. @param editor: An EDITOR value from the environment. @type: string @rtype: bool @return: True if an executable is found, False otherwise. """ editor_split = util.shlex_split(editor) if not editor_split: return False filename = editor_split[0] if not os.path.isabs(filename): return find_binary(filename) is not None return os.access(filename, os.X_OK) and os.path.isfile(filename)
def diffstatusoutput(cmd, file1, file2):
"""
Execute the string cmd in a shell with getstatusoutput() and return a
2-tuple (status, output).
"""
# Use Popen to emulate getstatusoutput(), since getstatusoutput() may
# raise a UnicodeDecodeError which makes the output inaccessible.
args = shlex_split(cmd % (file1, file2))
if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000:
# Python 3.1 does not support bytes in Popen args.
args = [portage._unicode_encode(x, errors='strict') for x in args]
proc = subprocess.Popen(args,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = portage._unicode_decode(proc.communicate()[0])
if output and output[-1] == "\n":
# getstatusoutput strips one newline
output = output[:-1]
return (proc.wait(), output)
def set_port_dirs(self, portdir, portdir_overlay): self.portdir = portdir self.portdir_overlay = portdir_overlay if self.portdir_overlay is None: self.portdir_overlay = "" self.overlay_profiles = [] for ov in shlex_split(self.portdir_overlay): ov = normalize_path(ov) profiles_dir = os.path.join(ov, "profiles") if os.path.isdir(profiles_dir): self.overlay_profiles.append(profiles_dir) self.profile_locations = [os.path.join(portdir, "profiles")] + self.overlay_profiles self.profile_and_user_locations = self.profile_locations[:] if self._user_config: self.profile_and_user_locations.append(self.abs_user_config) self.profile_locations = tuple(self.profile_locations) self.profile_and_user_locations = tuple(self.profile_and_user_locations)
def __init__(self, roots):
self._shown_repos = {}
self._unknown_repo = False
repo_paths = set()
for root_config in roots.values():
portdir = root_config.settings.get("PORTDIR")
if portdir:
repo_paths.add(portdir)
overlays = root_config.settings.get("PORTDIR_OVERLAY")
if overlays:
repo_paths.update(shlex_split(overlays))
repo_paths = list(repo_paths)
self._repo_paths = repo_paths
self._repo_paths_real = [os.path.realpath(repo_path) for repo_path in repo_paths]
# pre-allocate index for PORTDIR so that it always has index 0.
for root_config in roots.values():
portdb = root_config.trees["porttree"].dbapi
portdir = portdb.porttree_root
if portdir:
self.repoStr(portdir)
def __init__(self, roots):
self._shown_repos = {}
self._unknown_repo = False
repo_paths = set()
for root_config in roots.values():
portdir = root_config.settings.get("PORTDIR")
if portdir:
repo_paths.add(portdir)
overlays = root_config.settings.get("PORTDIR_OVERLAY")
if overlays:
repo_paths.update(shlex_split(overlays))
repo_paths = list(repo_paths)
self._repo_paths = repo_paths
self._repo_paths_real = [ os.path.realpath(repo_path) \
for repo_path in repo_paths ]
# pre-allocate index for PORTDIR so that it always has index 0.
for root_config in roots.values():
portdb = root_config.trees["porttree"].dbapi
portdir = portdb.porttree_root
if portdir:
self.repoStr(portdir)
def run(self):
config_protect = shlex_split(settings.get("CONFIG_PROTECT", ""))
self.chk_updated_cfg_files(settings["EROOT"], config_protect)
def ExtractKernelVersion(base_dir):
"""
Try to figure out what kernel version we are running
@param base_dir: Path to sources (usually /usr/src/linux)
@type base_dir: string
@rtype: tuple( version[string], error[string])
@return:
1. tuple( version[string], error[string])
Either version or error is populated (but never both)
"""
lines = []
pathname = os.path.join(base_dir, 'Makefile')
try:
f = io.open(_unicode_encode(pathname,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['content'],
errors='replace')
except OSError as details:
return (None, str(details))
except IOError as details:
return (None, str(details))
try:
for i in range(4):
lines.append(f.readline())
except OSError as details:
return (None, str(details))
except IOError as details:
return (None, str(details))
finally:
f.close()
lines = [l.strip() for l in lines]
version = ''
#XXX: The following code relies on the ordering of vars within the Makefile
for line in lines:
# split on the '=' then remove annoying whitespace
items = line.split("=")
items = [i.strip() for i in items]
if items[0] == 'VERSION' or \
items[0] == 'PATCHLEVEL':
version += items[1]
version += "."
elif items[0] == 'SUBLEVEL':
version += items[1]
elif items[0] == 'EXTRAVERSION' and \
items[-1] != items[0]:
version += items[1]
# Grab a list of files named localversion* and sort them
localversions = os.listdir(base_dir)
for x in range(len(localversions) - 1, -1, -1):
if localversions[x][:12] != "localversion":
del localversions[x]
localversions.sort()
# Append the contents of each to the version string, stripping ALL whitespace
for lv in localversions:
version += "".join(" ".join(grabfile(base_dir + "/" + lv)).split())
# Check the .config for a CONFIG_LOCALVERSION and append that too, also stripping whitespace
loader = KeyValuePairFileLoader(os.path.join(base_dir, ".config"), None)
kernelconfig, loader_errors = loader.load()
if loader_errors:
for file_path, file_errors in loader_errors.items():
for error_str in file_errors:
writemsg_level("%s: %s\n" % (file_path, error_str),
level=logging.ERROR,
noiselevel=-1)
if kernelconfig and "CONFIG_LOCALVERSION" in kernelconfig:
version += "".join(shlex_split(kernelconfig["CONFIG_LOCALVERSION"]))
return (version, None)
def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True):
root = trees._running_eroot
mysettings = trees[root]["vartree"].settings
portdb = trees[root]["porttree"].dbapi
vardb = trees[root]["vartree"].dbapi
bindb = trees[root]["bintree"].dbapi
world_file = os.path.join(mysettings['EROOT'], WORLD_FILE)
world_list = grabfile(world_file)
world_modified = False
world_warnings = set()
updpath_map = {}
# Maps repo_name to list of updates. If a given repo has no updates
# directory, it will be omitted. If a repo has an updates directory
# but none need to be applied (according to timestamp logic), the
# value in the dict will be an empty list.
repo_map = {}
timestamps = {}
retupd = False
update_notice_printed = False
for repo_name in portdb.getRepositories():
repo = portdb.getRepositoryPath(repo_name)
updpath = os.path.join(repo, "profiles", "updates")
if not os.path.isdir(updpath):
continue
if updpath in updpath_map:
repo_map[repo_name] = updpath_map[updpath]
continue
try:
if if_mtime_changed:
update_data = grab_updates(updpath, prev_mtimes=prev_mtimes)
else:
update_data = grab_updates(updpath)
except DirectoryNotFound:
continue
myupd = []
updpath_map[updpath] = myupd
repo_map[repo_name] = myupd
if len(update_data) > 0:
for mykey, mystat, mycontent in update_data:
if not update_notice_printed:
update_notice_printed = True
writemsg_stdout("\n")
if quiet:
writemsg_stdout(colorize("GOOD",
_("Performing Global Updates\n")))
writemsg_stdout(_("(Could take a couple of minutes if you have a lot of binary packages.)\n"))
else:
writemsg_stdout(colorize("GOOD",
_("Performing Global Updates:\n")))
writemsg_stdout(_("(Could take a couple of minutes if you have a lot of binary packages.)\n"))
writemsg_stdout(_(" %s='update pass' %s='binary update' "
"%s='/var/db update' %s='/var/db move'\n"
" %s='/var/db SLOT move' %s='binary move' "
"%s='binary SLOT move'\n %s='update /etc/portage/package.*'\n") % \
(bold("."), bold("*"), bold("#"), bold("@"), bold("s"), bold("%"), bold("S"), bold("p")))
valid_updates, errors = parse_updates(mycontent)
myupd.extend(valid_updates)
if not quiet:
writemsg_stdout(bold(mykey))
writemsg_stdout(len(valid_updates) * "." + "\n")
if len(errors) == 0:
# Update our internal mtime since we
# processed all of our directives.
timestamps[mykey] = mystat[stat.ST_MTIME]
else:
for msg in errors:
writemsg("%s\n" % msg, noiselevel=-1)
if myupd:
retupd = True
if retupd:
if os.access(bindb.bintree.pkgdir, os.W_OK):
# Call binarytree.populate(), since we want to make sure it's
# only populated with local packages here (getbinpkgs=0).
bindb.bintree.populate()
else:
bindb = None
master_repo = portdb.getRepositoryName(portdb.porttree_root)
if master_repo in repo_map:
repo_map['DEFAULT'] = repo_map[master_repo]
for repo_name, myupd in repo_map.items():
if repo_name == 'DEFAULT':
continue
if not myupd:
continue
def repo_match(repository):
return repository == repo_name or \
(repo_name == master_repo and repository not in repo_map)
def _world_repo_match(atoma, atomb):
"""
Check whether to perform a world change from atoma to atomb.
If best vardb match for atoma comes from the same repository
as the update file, allow that. Additionally, if portdb still
can find a match for old atom name, warn about that.
"""
matches = vardb.match(atoma)
if not matches:
matches = vardb.match(atomb)
if matches and \
repo_match(vardb.aux_get(best(matches), ['repository'])[0]):
if portdb.match(atoma):
world_warnings.add((atoma, atomb))
return True
else:
return False
for update_cmd in myupd:
for pos, atom in enumerate(world_list):
new_atom = update_dbentry(update_cmd, atom)
if atom != new_atom:
if _world_repo_match(atom, new_atom):
world_list[pos] = new_atom
world_modified = True
for update_cmd in myupd:
if update_cmd[0] == "move":
moves = vardb.move_ent(update_cmd, repo_match=repo_match)
if moves:
writemsg_stdout(moves * "@")
if bindb:
moves = bindb.move_ent(update_cmd, repo_match=repo_match)
if moves:
writemsg_stdout(moves * "%")
elif update_cmd[0] == "slotmove":
moves = vardb.move_slot_ent(update_cmd, repo_match=repo_match)
if moves:
writemsg_stdout(moves * "s")
if bindb:
moves = bindb.move_slot_ent(update_cmd, repo_match=repo_match)
if moves:
writemsg_stdout(moves * "S")
if world_modified:
world_list.sort()
write_atomic(world_file,
"".join("%s\n" % (x,) for x in world_list))
if world_warnings:
# XXX: print warning that we've updated world entries
# and the old name still matches something (from an overlay)?
pass
if retupd:
def _config_repo_match(repo_name, atoma, atomb):
"""
Check whether to perform a world change from atoma to atomb.
If best vardb match for atoma comes from the same repository
as the update file, allow that. Additionally, if portdb still
can find a match for old atom name, warn about that.
"""
matches = vardb.match(atoma)
if not matches:
matches = vardb.match(atomb)
if not matches:
return False
repository = vardb.aux_get(best(matches), ['repository'])[0]
return repository == repo_name or \
(repo_name == master_repo and repository not in repo_map)
update_config_files(root,
shlex_split(mysettings.get("CONFIG_PROTECT", "")),
shlex_split(mysettings.get("CONFIG_PROTECT_MASK", "")),
repo_map, match_callback=_config_repo_match)
# The above global updates proceed quickly, so they
# are considered a single mtimedb transaction.
if timestamps:
# We do not update the mtime in the mtimedb
# until after _all_ of the above updates have
# been processed because the mtimedb will
# automatically commit when killed by ctrl C.
for mykey, mtime in timestamps.items():
prev_mtimes[mykey] = mtime
do_upgrade_packagesmessage = False
# We gotta do the brute force updates for these now.
if True:
def onUpdate(maxval, curval):
if curval > 0:
writemsg_stdout("#")
if quiet:
onUpdate = None
vardb.update_ents(repo_map, onUpdate=onUpdate)
if bindb:
def onUpdate(maxval, curval):
if curval > 0:
writemsg_stdout("*")
if quiet:
onUpdate = None
bindb.update_ents(repo_map, onUpdate=onUpdate)
else:
do_upgrade_packagesmessage = 1
# Update progress above is indicated by characters written to stdout so
# we print a couple new lines here to separate the progress output from
# what follows.
writemsg_stdout("\n\n")
if do_upgrade_packagesmessage and bindb and \
bindb.cpv_all():
writemsg_stdout(_(" ** Skipping packages. Run 'fixpackages' or set it in FEATURES to fix the tbz2's in the packages directory.\n"))
writemsg_stdout(bold(_("Note: This can take a very long time.")))
writemsg_stdout("\n")
return retupd
def update(self):
''' Update existing git repository, and ignore the syncuri. We are
going to trust the user and assume that the user is in the branch
that he/she wants updated. We'll let the user manage branches with
git directly.
'''
if not self.has_bin:
return (1, False)
git_cmd_opts = ""
quiet = self.settings.get("PORTAGE_QUIET") == "1"
if self.repo.module_specific_options.get('sync-git-env'):
shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-env'])
env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(env)
if self.repo.module_specific_options.get('sync-git-pull-env'):
shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-pull-env'])
pull_env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(pull_env)
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
if self.repo.module_specific_options.get('sync-git-pull-extra-opts'):
git_cmd_opts += " %s" % self.repo.module_specific_options['sync-git-pull-extra-opts']
try:
remote_branch = portage._unicode_decode(
subprocess.check_output([self.bin_command, 'rev-parse',
'--abbrev-ref', '--symbolic-full-name', '@{upstream}'],
cwd=portage._unicode_encode(self.repo.location))).rstrip('\n')
except subprocess.CalledProcessError as e:
msg = "!!! git rev-parse error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (e.returncode, False)
shallow = self.repo.sync_depth is not None and self.repo.sync_depth != 0
if shallow:
git_cmd_opts += " --depth %d" % self.repo.sync_depth
# For shallow fetch, unreachable objects may need to be pruned
# manually, in order to prevent automatic git gc calls from
# eventually failing (see bug 599008).
gc_cmd = ['git', '-c', 'gc.autodetach=false', 'gc', '--auto']
if quiet:
gc_cmd.append('--quiet')
exitcode = subprocess.call(gc_cmd,
cwd=portage._unicode_encode(self.repo.location))
if exitcode != os.EX_OK:
msg = "!!! git gc error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
git_cmd = "%s fetch %s%s" % (self.bin_command,
remote_branch.partition('/')[0], git_cmd_opts)
writemsg_level(git_cmd + "\n")
rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
previous_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(self.repo.location))
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), git_cmd),
**self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! git fetch error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
if not self.verify_head(revision='refs/remotes/%s' % remote_branch):
return (1, False)
if shallow:
# Since the default merge strategy typically fails when
# the depth is not unlimited, `git reset --merge`.
merge_cmd = [self.bin_command, 'reset', '--merge']
else:
merge_cmd = [self.bin_command, 'merge']
merge_cmd.append('refs/remotes/%s' % remote_branch)
if quiet:
merge_cmd.append('--quiet')
exitcode = subprocess.call(merge_cmd,
cwd=portage._unicode_encode(self.repo.location))
if exitcode != os.EX_OK:
msg = "!!! git merge error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
current_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(self.repo.location))
return (os.EX_OK, current_rev != previous_rev)
def fetch(myuris, mysettings, listonly=0, fetchonly=0,
locks_in_subdir=".locks", use_locks=1, try_mirrors=1, digests=None,
allow_missing_digests=True, force=False):
"""
Fetch files to DISTDIR and also verify digests if they are available.
@param myuris: Maps each file name to a tuple of available fetch URIs.
@type myuris: dict
@param mysettings: Portage config instance.
@type mysettings: portage.config
@param listonly: Only print URIs and do not actually fetch them.
@type listonly: bool
@param fetchonly: Do not block for files that are locked by a
concurrent fetcher process. This means that the function can
return successfully *before* all files have been successfully
fetched!
@type fetchonly: bool
@param use_locks: Enable locks. This parameter is ineffective if
FEATURES=distlocks is disabled in the portage config!
@type use_locks: bool
@param digests: Maps each file name to a dict of digest types and values.
@type digests: dict
@param allow_missing_digests: Enable fetch even if there are no digests
available for verification.
@type allow_missing_digests: bool
@param force: Force download, even when a file already exists in
DISTDIR. This is most useful when there are no digests available,
since otherwise download will be automatically forced if the
existing file does not match the available digests. Also, this
avoids the need to remove the existing file in advance, which
makes it possible to atomically replace the file and avoid
interference with concurrent processes.
@type force: bool
@rtype: int
@return: 1 if successful, 0 otherwise.
"""
if force and digests:
# Since the force parameter can trigger unnecessary fetch when the
# digests match, do not allow force=True when digests are provided.
raise PortageException(_('fetch: force=True is not allowed when digests are provided'))
if not myuris:
return 1
features = mysettings.features
restrict = mysettings.get("PORTAGE_RESTRICT","").split()
userfetch = portage.data.secpass >= 2 and "userfetch" in features
# 'nomirror' is bad/negative logic. You Restrict mirroring, not no-mirroring.
restrict_mirror = "mirror" in restrict or "nomirror" in restrict
if restrict_mirror:
if ("mirror" in features) and ("lmirror" not in features):
# lmirror should allow you to bypass mirror restrictions.
# XXX: This is not a good thing, and is temporary at best.
print(_(">>> \"mirror\" mode desired and \"mirror\" restriction found; skipping fetch."))
return 1
# Generally, downloading the same file repeatedly from
# every single available mirror is a waste of bandwidth
# and time, so there needs to be a cap.
checksum_failure_max_tries = 5
v = checksum_failure_max_tries
try:
v = int(mysettings.get("PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS",
checksum_failure_max_tries))
except (ValueError, OverflowError):
writemsg(_("!!! Variable PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS"
" contains non-integer value: '%s'\n") % \
mysettings["PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS"], noiselevel=-1)
writemsg(_("!!! Using PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS "
"default value: %s\n") % checksum_failure_max_tries,
noiselevel=-1)
v = checksum_failure_max_tries
if v < 1:
writemsg(_("!!! Variable PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS"
" contains value less than 1: '%s'\n") % v, noiselevel=-1)
writemsg(_("!!! Using PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS "
"default value: %s\n") % checksum_failure_max_tries,
noiselevel=-1)
v = checksum_failure_max_tries
checksum_failure_max_tries = v
del v
fetch_resume_size_default = "350K"
fetch_resume_size = mysettings.get("PORTAGE_FETCH_RESUME_MIN_SIZE")
if fetch_resume_size is not None:
fetch_resume_size = "".join(fetch_resume_size.split())
if not fetch_resume_size:
# If it's undefined or empty, silently use the default.
fetch_resume_size = fetch_resume_size_default
match = _fetch_resume_size_re.match(fetch_resume_size)
if match is None or \
(match.group(2).upper() not in _size_suffix_map):
writemsg(_("!!! Variable PORTAGE_FETCH_RESUME_MIN_SIZE"
" contains an unrecognized format: '%s'\n") % \
mysettings["PORTAGE_FETCH_RESUME_MIN_SIZE"], noiselevel=-1)
writemsg(_("!!! Using PORTAGE_FETCH_RESUME_MIN_SIZE "
"default value: %s\n") % fetch_resume_size_default,
noiselevel=-1)
fetch_resume_size = None
if fetch_resume_size is None:
fetch_resume_size = fetch_resume_size_default
match = _fetch_resume_size_re.match(fetch_resume_size)
fetch_resume_size = int(match.group(1)) * \
2 ** _size_suffix_map[match.group(2).upper()]
# Behave like the package has RESTRICT="primaryuri" after a
# couple of checksum failures, to increase the probablility
# of success before checksum_failure_max_tries is reached.
checksum_failure_primaryuri = 2
thirdpartymirrors = mysettings.thirdpartymirrors()
# In the background parallel-fetch process, it's safe to skip checksum
# verification of pre-existing files in $DISTDIR that have the correct
# file size. The parent process will verify their checksums prior to
# the unpack phase.
parallel_fetchonly = "PORTAGE_PARALLEL_FETCHONLY" in mysettings
if parallel_fetchonly:
fetchonly = 1
check_config_instance(mysettings)
custommirrors = grabdict(os.path.join(mysettings["PORTAGE_CONFIGROOT"],
CUSTOM_MIRRORS_FILE), recursive=1)
mymirrors=[]
if listonly or ("distlocks" not in features):
use_locks = 0
distdir_writable = os.access(mysettings["DISTDIR"], os.W_OK)
fetch_to_ro = 0
if "skiprocheck" in features:
fetch_to_ro = 1
if not distdir_writable and fetch_to_ro:
if use_locks:
writemsg(colorize("BAD",
_("!!! For fetching to a read-only filesystem, "
"locking should be turned off.\n")), noiselevel=-1)
writemsg(_("!!! This can be done by adding -distlocks to "
"FEATURES in /etc/portage/make.conf\n"), noiselevel=-1)
# use_locks = 0
# local mirrors are always added
if try_mirrors and "local" in custommirrors:
mymirrors += custommirrors["local"]
if restrict_mirror:
# We don't add any mirrors.
pass
else:
if try_mirrors:
mymirrors += [x.rstrip("/") for x in mysettings["GENTOO_MIRRORS"].split() if x]
hash_filter = _hash_filter(mysettings.get("PORTAGE_CHECKSUM_FILTER", ""))
if hash_filter.transparent:
hash_filter = None
skip_manifest = mysettings.get("EBUILD_SKIP_MANIFEST") == "1"
if skip_manifest:
allow_missing_digests = True
pkgdir = mysettings.get("O")
if digests is None and not (pkgdir is None or skip_manifest):
mydigests = mysettings.repositories.get_repo_for_location(
os.path.dirname(os.path.dirname(pkgdir))).load_manifest(
pkgdir, mysettings["DISTDIR"]).getTypeDigests("DIST")
elif digests is None or skip_manifest:
# no digests because fetch was not called for a specific package
mydigests = {}
else:
mydigests = digests
ro_distdirs = [x for x in \
shlex_split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \
if os.path.isdir(x)]
fsmirrors = []
for x in range(len(mymirrors)-1,-1,-1):
if mymirrors[x] and mymirrors[x][0]=='/':
fsmirrors += [mymirrors[x]]
del mymirrors[x]
restrict_fetch = "fetch" in restrict
force_mirror = "force-mirror" in features and not restrict_mirror
custom_local_mirrors = custommirrors.get("local", [])
if restrict_fetch:
# With fetch restriction, a normal uri may only be fetched from
# custom local mirrors (if available). A mirror:// uri may also
# be fetched from specific mirrors (effectively overriding fetch
# restriction, but only for specific mirrors).
locations = custom_local_mirrors
else:
locations = mymirrors
file_uri_tuples = []
# Check for 'items' attribute since OrderedDict is not a dict.
if hasattr(myuris, 'items'):
for myfile, uri_set in myuris.items():
for myuri in uri_set:
file_uri_tuples.append((myfile, myuri))
if not uri_set:
file_uri_tuples.append((myfile, None))
else:
for myuri in myuris:
if urlparse(myuri).scheme:
file_uri_tuples.append((os.path.basename(myuri), myuri))
else:
file_uri_tuples.append((os.path.basename(myuri), None))
filedict = OrderedDict()
primaryuri_dict = {}
thirdpartymirror_uris = {}
for myfile, myuri in file_uri_tuples:
if myfile not in filedict:
filedict[myfile]=[]
if distdir_writable:
mirror_cache = os.path.join(mysettings["DISTDIR"],
".mirror-cache.json")
else:
mirror_cache = None
for l in locations:
filedict[myfile].append(functools.partial(
get_mirror_url, l, myfile, mysettings, mirror_cache))
if myuri is None:
continue
if myuri[:9]=="mirror://":
eidx = myuri.find("/", 9)
if eidx != -1:
mirrorname = myuri[9:eidx]
path = myuri[eidx+1:]
# Try user-defined mirrors first
if mirrorname in custommirrors:
for cmirr in custommirrors[mirrorname]:
filedict[myfile].append(
cmirr.rstrip("/") + "/" + path)
# now try the official mirrors
if mirrorname in thirdpartymirrors:
uris = [locmirr.rstrip("/") + "/" + path \
for locmirr in thirdpartymirrors[mirrorname]]
random.shuffle(uris)
filedict[myfile].extend(uris)
thirdpartymirror_uris.setdefault(myfile, []).extend(uris)
if mirrorname not in custommirrors and \
mirrorname not in thirdpartymirrors:
writemsg(_("!!! No known mirror by the name: %s\n") % (mirrorname))
else:
writemsg(_("Invalid mirror definition in SRC_URI:\n"), noiselevel=-1)
writemsg(" %s\n" % (myuri), noiselevel=-1)
else:
if restrict_fetch or force_mirror:
# Only fetch from specific mirrors is allowed.
continue
primaryuris = primaryuri_dict.get(myfile)
if primaryuris is None:
primaryuris = []
primaryuri_dict[myfile] = primaryuris
primaryuris.append(myuri)
# Order primaryuri_dict values to match that in SRC_URI.
for uris in primaryuri_dict.values():
uris.reverse()
# Prefer thirdpartymirrors over normal mirrors in cases when
# the file does not yet exist on the normal mirrors.
for myfile, uris in thirdpartymirror_uris.items():
primaryuri_dict.setdefault(myfile, []).extend(uris)
# Now merge primaryuri values into filedict (includes mirrors
# explicitly referenced in SRC_URI).
if "primaryuri" in restrict:
for myfile, uris in filedict.items():
filedict[myfile] = primaryuri_dict.get(myfile, []) + uris
else:
for myfile in filedict:
filedict[myfile] += primaryuri_dict.get(myfile, [])
can_fetch=True
if listonly:
can_fetch = False
if can_fetch and not fetch_to_ro:
try:
_ensure_distdir(mysettings, mysettings["DISTDIR"])
except PortageException as e:
if not os.path.isdir(mysettings["DISTDIR"]):
writemsg("!!! %s\n" % str(e), noiselevel=-1)
writemsg(_("!!! Directory Not Found: DISTDIR='%s'\n") % mysettings["DISTDIR"], noiselevel=-1)
writemsg(_("!!! Fetching will fail!\n"), noiselevel=-1)
if can_fetch and \
not fetch_to_ro and \
not os.access(mysettings["DISTDIR"], os.W_OK):
writemsg(_("!!! No write access to '%s'\n") % mysettings["DISTDIR"],
noiselevel=-1)
can_fetch = False
distdir_writable = can_fetch and not fetch_to_ro
failed_files = set()
restrict_fetch_msg = False
valid_hashes = set(get_valid_checksum_keys())
valid_hashes.discard("size")
for myfile in filedict:
"""
fetched status
0 nonexistent
1 partially downloaded
2 completely downloaded
"""
fetched = 0
orig_digests = mydigests.get(myfile, {})
if not (allow_missing_digests or listonly):
verifiable_hash_types = set(orig_digests).intersection(valid_hashes)
if not verifiable_hash_types:
expected = " ".join(sorted(valid_hashes))
got = set(orig_digests)
got.discard("size")
got = " ".join(sorted(got))
reason = (_("Insufficient data for checksum verification"),
got, expected)
writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile,
noiselevel=-1)
writemsg(_("!!! Reason: %s\n") % reason[0],
noiselevel=-1)
writemsg(_("!!! Got: %s\n!!! Expected: %s\n") % \
(reason[1], reason[2]), noiselevel=-1)
if fetchonly:
failed_files.add(myfile)
continue
else:
return 0
size = orig_digests.get("size")
if size == 0:
# Zero-byte distfiles are always invalid, so discard their digests.
del mydigests[myfile]
orig_digests.clear()
size = None
pruned_digests = orig_digests
if parallel_fetchonly:
pruned_digests = {}
if size is not None:
pruned_digests["size"] = size
myfile_path = os.path.join(mysettings["DISTDIR"], myfile)
download_path = myfile_path if fetch_to_ro else myfile_path + _download_suffix
has_space = True
has_space_superuser = True
file_lock = None
if listonly:
writemsg_stdout("\n", noiselevel=-1)
else:
# check if there is enough space in DISTDIR to completely store myfile
# overestimate the filesize so we aren't bitten by FS overhead
vfs_stat = None
if size is not None and hasattr(os, "statvfs"):
try:
vfs_stat = os.statvfs(mysettings["DISTDIR"])
except OSError as e:
writemsg_level("!!! statvfs('%s'): %s\n" %
(mysettings["DISTDIR"], e),
noiselevel=-1, level=logging.ERROR)
del e
if vfs_stat is not None:
try:
mysize = os.stat(myfile_path).st_size
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
mysize = 0
if (size - mysize + vfs_stat.f_bsize) >= \
(vfs_stat.f_bsize * vfs_stat.f_bavail):
if (size - mysize + vfs_stat.f_bsize) >= \
(vfs_stat.f_bsize * vfs_stat.f_bfree):
has_space_superuser = False
if not has_space_superuser:
has_space = False
elif portage.data.secpass < 2:
has_space = False
elif userfetch:
has_space = False
if distdir_writable and use_locks:
lock_kwargs = {}
if fetchonly:
lock_kwargs["flags"] = os.O_NONBLOCK
try:
file_lock = lockfile(myfile_path,
wantnewlockfile=1, **lock_kwargs)
except TryAgain:
writemsg(_(">>> File '%s' is already locked by "
"another fetcher. Continuing...\n") % myfile,
noiselevel=-1)
continue
try:
if not listonly:
eout = EOutput()
eout.quiet = mysettings.get("PORTAGE_QUIET") == "1"
match, mystat = _check_distfile(
myfile_path, pruned_digests, eout, hash_filter=hash_filter)
if match and not force:
# Skip permission adjustment for symlinks, since we don't
# want to modify anything outside of the primary DISTDIR,
# and symlinks typically point to PORTAGE_RO_DISTDIRS.
if distdir_writable and not os.path.islink(myfile_path):
try:
apply_secpass_permissions(myfile_path,
gid=portage_gid, mode=0o664, mask=0o2,
stat_cached=mystat)
except PortageException as e:
if not os.access(myfile_path, os.R_OK):
writemsg(_("!!! Failed to adjust permissions:"
" %s\n") % str(e), noiselevel=-1)
del e
continue
# Remove broken symlinks or symlinks to files which
# _check_distfile did not match above.
if distdir_writable and mystat is None or os.path.islink(myfile_path):
try:
os.unlink(myfile_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
mystat = None
if mystat is not None:
if stat.S_ISDIR(mystat.st_mode):
writemsg_level(
_("!!! Unable to fetch file since "
"a directory is in the way: \n"
"!!! %s\n") % myfile_path,
level=logging.ERROR, noiselevel=-1)
return 0
if distdir_writable and not force:
# Since _check_distfile did not match above, the file
# is either corrupt or its identity has changed since
# the last time it was fetched, so rename it.
temp_filename = _checksum_failure_temp_file(
mysettings, mysettings["DISTDIR"], myfile)
writemsg_stdout(_("Refetching... "
"File renamed to '%s'\n\n") % \
temp_filename, noiselevel=-1)
# Stat the temporary download file for comparison with
# fetch_resume_size.
try:
mystat = os.stat(download_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
mystat = None
if mystat is not None:
if mystat.st_size == 0:
if distdir_writable:
try:
os.unlink(download_path)
except OSError:
pass
elif distdir_writable and size is not None:
if mystat.st_size < fetch_resume_size and \
mystat.st_size < size:
# If the file already exists and the size does not
# match the existing digests, it may be that the
# user is attempting to update the digest. In this
# case, the digestgen() function will advise the
# user to use `ebuild --force foo.ebuild manifest`
# in order to force the old digests to be replaced.
# Since the user may want to keep this file, rename
# it instead of deleting it.
writemsg(_(">>> Renaming distfile with size "
"%d (smaller than " "PORTAGE_FETCH_RESU"
"ME_MIN_SIZE)\n") % mystat.st_size)
temp_filename = \
_checksum_failure_temp_file(
mysettings, mysettings["DISTDIR"],
os.path.basename(download_path))
writemsg_stdout(_("Refetching... "
"File renamed to '%s'\n\n") % \
temp_filename, noiselevel=-1)
elif mystat.st_size >= size:
temp_filename = \
_checksum_failure_temp_file(
mysettings, mysettings["DISTDIR"],
os.path.basename(download_path))
writemsg_stdout(_("Refetching... "
"File renamed to '%s'\n\n") % \
temp_filename, noiselevel=-1)
if distdir_writable and ro_distdirs:
readonly_file = None
for x in ro_distdirs:
filename = os.path.join(x, myfile)
match, mystat = _check_distfile(
filename, pruned_digests, eout, hash_filter=hash_filter)
if match:
readonly_file = filename
break
if readonly_file is not None:
try:
os.unlink(myfile_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
os.symlink(readonly_file, myfile_path)
continue
# this message is shown only after we know that
# the file is not already fetched
if not has_space:
writemsg(_("!!! Insufficient space to store %s in %s\n") % \
(myfile, mysettings["DISTDIR"]), noiselevel=-1)
if has_space_superuser:
writemsg(_("!!! Insufficient privileges to use "
"remaining space.\n"), noiselevel=-1)
if userfetch:
writemsg(_("!!! You may set FEATURES=\"-userfetch\""
" in /etc/portage/make.conf in order to fetch with\n"
"!!! superuser privileges.\n"), noiselevel=-1)
if fsmirrors and not os.path.exists(myfile_path) and has_space:
for mydir in fsmirrors:
mirror_file = os.path.join(mydir, myfile)
try:
shutil.copyfile(mirror_file, download_path)
writemsg(_("Local mirror has file: %s\n") % myfile)
break
except (IOError, OSError) as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
try:
mystat = os.stat(download_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
else:
# Skip permission adjustment for symlinks, since we don't
# want to modify anything outside of the primary DISTDIR,
# and symlinks typically point to PORTAGE_RO_DISTDIRS.
if not os.path.islink(download_path):
try:
apply_secpass_permissions(download_path,
gid=portage_gid, mode=0o664, mask=0o2,
stat_cached=mystat)
except PortageException as e:
if not os.access(download_path, os.R_OK):
writemsg(_("!!! Failed to adjust permissions:"
" %s\n") % (e,), noiselevel=-1)
# If the file is empty then it's obviously invalid. Remove
# the empty file and try to download if possible.
if mystat.st_size == 0:
if distdir_writable:
try:
os.unlink(download_path)
except EnvironmentError:
pass
elif not orig_digests:
# We don't have a digest, but the file exists. We must
# assume that it is fully downloaded.
if not force:
continue
else:
if (mydigests[myfile].get("size") is not None
and mystat.st_size < mydigests[myfile]["size"]
and not restrict_fetch):
fetched = 1 # Try to resume this download.
elif parallel_fetchonly and \
mystat.st_size == mydigests[myfile]["size"]:
eout = EOutput()
eout.quiet = \
mysettings.get("PORTAGE_QUIET") == "1"
eout.ebegin(
"%s size ;-)" % (myfile, ))
eout.eend(0)
continue
else:
digests = _filter_unaccelarated_hashes(mydigests[myfile])
if hash_filter is not None:
digests = _apply_hash_filter(digests, hash_filter)
verified_ok, reason = verify_all(download_path, digests)
if not verified_ok:
writemsg(_("!!! Previously fetched"
" file: '%s'\n") % myfile, noiselevel=-1)
writemsg(_("!!! Reason: %s\n") % reason[0],
noiselevel=-1)
writemsg(_("!!! Got: %s\n"
"!!! Expected: %s\n") % \
(reason[1], reason[2]), noiselevel=-1)
if reason[0] == _("Insufficient data for checksum verification"):
return 0
if distdir_writable:
temp_filename = \
_checksum_failure_temp_file(
mysettings, mysettings["DISTDIR"],
os.path.basename(download_path))
writemsg_stdout(_("Refetching... "
"File renamed to '%s'\n\n") % \
temp_filename, noiselevel=-1)
else:
if not fetch_to_ro:
_movefile(download_path, myfile_path, mysettings=mysettings)
eout = EOutput()
eout.quiet = \
mysettings.get("PORTAGE_QUIET", None) == "1"
if digests:
digests = list(digests)
digests.sort()
eout.ebegin(
"%s %s ;-)" % (myfile, " ".join(digests)))
eout.eend(0)
continue # fetch any remaining files
# Create a reversed list since that is optimal for list.pop().
uri_list = filedict[myfile][:]
uri_list.reverse()
checksum_failure_count = 0
tried_locations = set()
while uri_list:
loc = uri_list.pop()
if isinstance(loc, functools.partial):
loc = loc()
# Eliminate duplicates here in case we've switched to
# "primaryuri" mode on the fly due to a checksum failure.
if loc in tried_locations:
continue
tried_locations.add(loc)
if listonly:
writemsg_stdout(loc+" ", noiselevel=-1)
continue
# allow different fetchcommands per protocol
protocol = loc[0:loc.find("://")]
global_config_path = GLOBAL_CONFIG_PATH
if portage.const.EPREFIX:
global_config_path = os.path.join(portage.const.EPREFIX,
GLOBAL_CONFIG_PATH.lstrip(os.sep))
missing_file_param = False
fetchcommand_var = "FETCHCOMMAND_" + protocol.upper()
fetchcommand = mysettings.get(fetchcommand_var)
if fetchcommand is None:
fetchcommand_var = "FETCHCOMMAND"
fetchcommand = mysettings.get(fetchcommand_var)
if fetchcommand is None:
writemsg_level(
_("!!! %s is unset. It should "
"have been defined in\n!!! %s/make.globals.\n") \
% (fetchcommand_var, global_config_path),
level=logging.ERROR, noiselevel=-1)
return 0
if "${FILE}" not in fetchcommand:
writemsg_level(
_("!!! %s does not contain the required ${FILE}"
" parameter.\n") % fetchcommand_var,
level=logging.ERROR, noiselevel=-1)
missing_file_param = True
resumecommand_var = "RESUMECOMMAND_" + protocol.upper()
resumecommand = mysettings.get(resumecommand_var)
if resumecommand is None:
resumecommand_var = "RESUMECOMMAND"
resumecommand = mysettings.get(resumecommand_var)
if resumecommand is None:
writemsg_level(
_("!!! %s is unset. It should "
"have been defined in\n!!! %s/make.globals.\n") \
% (resumecommand_var, global_config_path),
level=logging.ERROR, noiselevel=-1)
return 0
if "${FILE}" not in resumecommand:
writemsg_level(
_("!!! %s does not contain the required ${FILE}"
" parameter.\n") % resumecommand_var,
level=logging.ERROR, noiselevel=-1)
missing_file_param = True
if missing_file_param:
writemsg_level(
_("!!! Refer to the make.conf(5) man page for "
"information about how to\n!!! correctly specify "
"FETCHCOMMAND and RESUMECOMMAND.\n"),
level=logging.ERROR, noiselevel=-1)
if myfile != os.path.basename(loc):
return 0
if not can_fetch:
if fetched != 2:
try:
mysize = os.stat(download_path).st_size
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
mysize = 0
if mysize == 0:
writemsg(_("!!! File %s isn't fetched but unable to get it.\n") % myfile,
noiselevel=-1)
elif size is None or size > mysize:
writemsg(_("!!! File %s isn't fully fetched, but unable to complete it\n") % myfile,
noiselevel=-1)
else:
writemsg(_("!!! File %s is incorrect size, "
"but unable to retry.\n") % myfile, noiselevel=-1)
return 0
else:
continue
if fetched != 2 and has_space:
#we either need to resume or start the download
if fetched == 1:
try:
mystat = os.stat(download_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
fetched = 0
else:
if distdir_writable and mystat.st_size < fetch_resume_size:
writemsg(_(">>> Deleting distfile with size "
"%d (smaller than " "PORTAGE_FETCH_RESU"
"ME_MIN_SIZE)\n") % mystat.st_size)
try:
os.unlink(download_path)
except OSError as e:
if e.errno not in \
(errno.ENOENT, errno.ESTALE):
raise
del e
fetched = 0
if fetched == 1:
#resume mode:
writemsg(_(">>> Resuming download...\n"))
locfetch=resumecommand
command_var = resumecommand_var
else:
#normal mode:
locfetch=fetchcommand
command_var = fetchcommand_var
writemsg_stdout(_(">>> Downloading '%s'\n") % \
_hide_url_passwd(loc))
variables = {
"URI": loc,
"FILE": os.path.basename(download_path)
}
for k in ("DISTDIR", "PORTAGE_SSH_OPTS"):
v = mysettings.get(k)
if v is not None:
variables[k] = v
myfetch = shlex_split(locfetch)
myfetch = [varexpand(x, mydict=variables) for x in myfetch]
myret = -1
try:
myret = _spawn_fetch(mysettings, myfetch)
finally:
try:
apply_secpass_permissions(download_path,
gid=portage_gid, mode=0o664, mask=0o2)
except FileNotFound:
pass
except PortageException as e:
if not os.access(download_path, os.R_OK):
writemsg(_("!!! Failed to adjust permissions:"
" %s\n") % str(e), noiselevel=-1)
del e
# If the file is empty then it's obviously invalid. Don't
# trust the return value from the fetcher. Remove the
# empty file and try to download again.
try:
mystat = os.lstat(download_path)
if mystat.st_size == 0 or (stat.S_ISLNK(mystat.st_mode) and not os.path.exists(download_path)):
os.unlink(download_path)
fetched = 0
continue
except EnvironmentError:
pass
if mydigests is not None and myfile in mydigests:
try:
mystat = os.stat(download_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
fetched = 0
else:
if stat.S_ISDIR(mystat.st_mode):
# This can happen if FETCHCOMMAND erroneously
# contains wget's -P option where it should
# instead have -O.
writemsg_level(
_("!!! The command specified in the "
"%s variable appears to have\n!!! "
"created a directory instead of a "
"normal file.\n") % command_var,
level=logging.ERROR, noiselevel=-1)
writemsg_level(
_("!!! Refer to the make.conf(5) "
"man page for information about how "
"to\n!!! correctly specify "
"FETCHCOMMAND and RESUMECOMMAND.\n"),
level=logging.ERROR, noiselevel=-1)
return 0
# no exception? file exists. let digestcheck() report
# an appropriately for size or checksum errors
# If the fetcher reported success and the file is
# too small, it's probably because the digest is
# bad (upstream changed the distfile). In this
# case we don't want to attempt to resume. Show a
# digest verification failure to that the user gets
# a clue about what just happened.
if myret != os.EX_OK and \
mystat.st_size < mydigests[myfile]["size"]:
# Fetch failed... Try the next one... Kill 404 files though.
if (mystat[stat.ST_SIZE]<100000) and (len(myfile)>4) and not ((myfile[-5:]==".html") or (myfile[-4:]==".htm")):
html404=re.compile("<title>.*(not found|404).*</title>",re.I|re.M)
with io.open(
_unicode_encode(download_path,
encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['content'], errors='replace'
) as f:
if html404.search(f.read()):
try:
os.unlink(download_path)
writemsg(_(">>> Deleting invalid distfile. (Improper 404 redirect from server.)\n"))
fetched = 0
continue
except (IOError, OSError):
pass
fetched = 1
continue
if True:
# File is the correct size--check the checksums for the fetched
# file NOW, for those users who don't have a stable/continuous
# net connection. This way we have a chance to try to download
# from another mirror...
digests = _filter_unaccelarated_hashes(mydigests[myfile])
if hash_filter is not None:
digests = _apply_hash_filter(digests, hash_filter)
verified_ok, reason = verify_all(download_path, digests)
if not verified_ok:
writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile,
noiselevel=-1)
writemsg(_("!!! Reason: %s\n") % reason[0],
noiselevel=-1)
writemsg(_("!!! Got: %s\n!!! Expected: %s\n") % \
(reason[1], reason[2]), noiselevel=-1)
if reason[0] == _("Insufficient data for checksum verification"):
return 0
if distdir_writable:
temp_filename = \
_checksum_failure_temp_file(
mysettings, mysettings["DISTDIR"],
os.path.basename(download_path))
writemsg_stdout(_("Refetching... "
"File renamed to '%s'\n\n") % \
temp_filename, noiselevel=-1)
fetched=0
checksum_failure_count += 1
if checksum_failure_count == \
checksum_failure_primaryuri:
# Switch to "primaryuri" mode in order
# to increase the probablility of
# of success.
primaryuris = \
primaryuri_dict.get(myfile)
if primaryuris:
uri_list.extend(
reversed(primaryuris))
if checksum_failure_count >= \
checksum_failure_max_tries:
break
else:
if not fetch_to_ro:
_movefile(download_path, myfile_path, mysettings=mysettings)
eout = EOutput()
eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1"
if digests:
eout.ebegin("%s %s ;-)" % \
(myfile, " ".join(sorted(digests))))
eout.eend(0)
fetched=2
break
else: # no digests available
if not myret:
if not fetch_to_ro:
_movefile(download_path, myfile_path, mysettings=mysettings)
fetched=2
break
elif mydigests!=None:
writemsg(_("No digest file available and download failed.\n\n"),
noiselevel=-1)
finally:
if use_locks and file_lock:
unlockfile(file_lock)
file_lock = None
if listonly:
writemsg_stdout("\n", noiselevel=-1)
if fetched != 2:
if restrict_fetch and not restrict_fetch_msg:
restrict_fetch_msg = True
msg = _("\n!!! %s/%s"
" has fetch restriction turned on.\n"
"!!! This probably means that this "
"ebuild's files must be downloaded\n"
"!!! manually. See the comments in"
" the ebuild for more information.\n\n") % \
(mysettings["CATEGORY"], mysettings["PF"])
writemsg_level(msg,
level=logging.ERROR, noiselevel=-1)
elif restrict_fetch:
pass
elif listonly:
pass
elif not filedict[myfile]:
writemsg(_("Warning: No mirrors available for file"
" '%s'\n") % (myfile), noiselevel=-1)
else:
writemsg(_("!!! Couldn't download '%s'. Aborting.\n") % myfile,
noiselevel=-1)
if listonly:
failed_files.add(myfile)
continue
elif fetchonly:
failed_files.add(myfile)
continue
return 0
if failed_files:
return 0
return 1
def fetch(myuris, mysettings, listonly=0, fetchonly=0,
locks_in_subdir=".locks", use_locks=1, try_mirrors=1, digests=None,
allow_missing_digests=True):
"fetch files. Will use digest file if available."
if not myuris:
return 1
features = mysettings.features
restrict = mysettings.get("PORTAGE_RESTRICT","").split()
userfetch = secpass >= 2 and "userfetch" in features
userpriv = secpass >= 2 and "userpriv" in features
# 'nomirror' is bad/negative logic. You Restrict mirroring, not no-mirroring.
restrict_mirror = "mirror" in restrict or "nomirror" in restrict
if restrict_mirror:
if ("mirror" in features) and ("lmirror" not in features):
# lmirror should allow you to bypass mirror restrictions.
# XXX: This is not a good thing, and is temporary at best.
print(_(">>> \"mirror\" mode desired and \"mirror\" restriction found; skipping fetch."))
return 1
# Generally, downloading the same file repeatedly from
# every single available mirror is a waste of bandwidth
# and time, so there needs to be a cap.
checksum_failure_max_tries = 5
v = checksum_failure_max_tries
try:
v = int(mysettings.get("PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS",
checksum_failure_max_tries))
except (ValueError, OverflowError):
writemsg(_("!!! Variable PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS"
" contains non-integer value: '%s'\n") % \
mysettings["PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS"], noiselevel=-1)
writemsg(_("!!! Using PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS "
"default value: %s\n") % checksum_failure_max_tries,
noiselevel=-1)
v = checksum_failure_max_tries
if v < 1:
writemsg(_("!!! Variable PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS"
" contains value less than 1: '%s'\n") % v, noiselevel=-1)
writemsg(_("!!! Using PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS "
"default value: %s\n") % checksum_failure_max_tries,
noiselevel=-1)
v = checksum_failure_max_tries
checksum_failure_max_tries = v
del v
fetch_resume_size_default = "350K"
fetch_resume_size = mysettings.get("PORTAGE_FETCH_RESUME_MIN_SIZE")
if fetch_resume_size is not None:
fetch_resume_size = "".join(fetch_resume_size.split())
if not fetch_resume_size:
# If it's undefined or empty, silently use the default.
fetch_resume_size = fetch_resume_size_default
match = _fetch_resume_size_re.match(fetch_resume_size)
if match is None or \
(match.group(2).upper() not in _size_suffix_map):
writemsg(_("!!! Variable PORTAGE_FETCH_RESUME_MIN_SIZE"
" contains an unrecognized format: '%s'\n") % \
mysettings["PORTAGE_FETCH_RESUME_MIN_SIZE"], noiselevel=-1)
writemsg(_("!!! Using PORTAGE_FETCH_RESUME_MIN_SIZE "
"default value: %s\n") % fetch_resume_size_default,
noiselevel=-1)
fetch_resume_size = None
if fetch_resume_size is None:
fetch_resume_size = fetch_resume_size_default
match = _fetch_resume_size_re.match(fetch_resume_size)
fetch_resume_size = int(match.group(1)) * \
2 ** _size_suffix_map[match.group(2).upper()]
# Behave like the package has RESTRICT="primaryuri" after a
# couple of checksum failures, to increase the probablility
# of success before checksum_failure_max_tries is reached.
checksum_failure_primaryuri = 2
thirdpartymirrors = mysettings.thirdpartymirrors()
# In the background parallel-fetch process, it's safe to skip checksum
# verification of pre-existing files in $DISTDIR that have the correct
# file size. The parent process will verify their checksums prior to
# the unpack phase.
parallel_fetchonly = "PORTAGE_PARALLEL_FETCHONLY" in mysettings
if parallel_fetchonly:
fetchonly = 1
check_config_instance(mysettings)
custommirrors = grabdict(os.path.join(mysettings["PORTAGE_CONFIGROOT"],
CUSTOM_MIRRORS_FILE), recursive=1)
mymirrors=[]
if listonly or ("distlocks" not in features):
use_locks = 0
fetch_to_ro = 0
if "skiprocheck" in features:
fetch_to_ro = 1
if not os.access(mysettings["DISTDIR"],os.W_OK) and fetch_to_ro:
if use_locks:
writemsg(colorize("BAD",
_("!!! For fetching to a read-only filesystem, "
"locking should be turned off.\n")), noiselevel=-1)
writemsg(_("!!! This can be done by adding -distlocks to "
"FEATURES in /etc/make.conf\n"), noiselevel=-1)
# use_locks = 0
# local mirrors are always added
if "local" in custommirrors:
mymirrors += custommirrors["local"]
if restrict_mirror:
# We don't add any mirrors.
pass
else:
if try_mirrors:
mymirrors += [x.rstrip("/") for x in mysettings["GENTOO_MIRRORS"].split() if x]
hash_filter = _hash_filter(mysettings.get("PORTAGE_CHECKSUM_FILTER", ""))
if hash_filter.transparent:
hash_filter = None
skip_manifest = mysettings.get("EBUILD_SKIP_MANIFEST") == "1"
if skip_manifest:
allow_missing_digests = True
pkgdir = mysettings.get("O")
if digests is None and not (pkgdir is None or skip_manifest):
mydigests = mysettings.repositories.get_repo_for_location(
os.path.dirname(os.path.dirname(pkgdir))).load_manifest(
pkgdir, mysettings["DISTDIR"]).getTypeDigests("DIST")
elif digests is None or skip_manifest:
# no digests because fetch was not called for a specific package
mydigests = {}
else:
mydigests = digests
ro_distdirs = [x for x in \
shlex_split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \
if os.path.isdir(x)]
fsmirrors = []
for x in range(len(mymirrors)-1,-1,-1):
if mymirrors[x] and mymirrors[x][0]=='/':
fsmirrors += [mymirrors[x]]
del mymirrors[x]
restrict_fetch = "fetch" in restrict
force_mirror = "force-mirror" in features and not restrict_mirror
custom_local_mirrors = custommirrors.get("local", [])
if restrict_fetch:
# With fetch restriction, a normal uri may only be fetched from
# custom local mirrors (if available). A mirror:// uri may also
# be fetched from specific mirrors (effectively overriding fetch
# restriction, but only for specific mirrors).
locations = custom_local_mirrors
else:
locations = mymirrors
file_uri_tuples = []
# Check for 'items' attribute since OrderedDict is not a dict.
if hasattr(myuris, 'items'):
for myfile, uri_set in myuris.items():
for myuri in uri_set:
file_uri_tuples.append((myfile, myuri))
else:
for myuri in myuris:
file_uri_tuples.append((os.path.basename(myuri), myuri))
filedict = OrderedDict()
primaryuri_dict = {}
thirdpartymirror_uris = {}
for myfile, myuri in file_uri_tuples:
if myfile not in filedict:
filedict[myfile]=[]
for y in range(0,len(locations)):
filedict[myfile].append(locations[y]+"/distfiles/"+myfile)
if myuri[:9]=="mirror://":
eidx = myuri.find("/", 9)
if eidx != -1:
mirrorname = myuri[9:eidx]
path = myuri[eidx+1:]
# Try user-defined mirrors first
if mirrorname in custommirrors:
for cmirr in custommirrors[mirrorname]:
filedict[myfile].append(
cmirr.rstrip("/") + "/" + path)
# now try the official mirrors
if mirrorname in thirdpartymirrors:
uris = [locmirr.rstrip("/") + "/" + path \
for locmirr in thirdpartymirrors[mirrorname]]
random.shuffle(uris)
filedict[myfile].extend(uris)
thirdpartymirror_uris.setdefault(myfile, []).extend(uris)
if not filedict[myfile]:
writemsg(_("No known mirror by the name: %s\n") % (mirrorname))
else:
writemsg(_("Invalid mirror definition in SRC_URI:\n"), noiselevel=-1)
writemsg(" %s\n" % (myuri), noiselevel=-1)
else:
if restrict_fetch or force_mirror:
# Only fetch from specific mirrors is allowed.
continue
primaryuris = primaryuri_dict.get(myfile)
if primaryuris is None:
primaryuris = []
primaryuri_dict[myfile] = primaryuris
primaryuris.append(myuri)
# Order primaryuri_dict values to match that in SRC_URI.
for uris in primaryuri_dict.values():
uris.reverse()
# Prefer thirdpartymirrors over normal mirrors in cases when
# the file does not yet exist on the normal mirrors.
for myfile, uris in thirdpartymirror_uris.items():
primaryuri_dict.setdefault(myfile, []).extend(uris)
# Now merge primaryuri values into filedict (includes mirrors
# explicitly referenced in SRC_URI).
if "primaryuri" in restrict:
for myfile, uris in filedict.items():
filedict[myfile] = primaryuri_dict.get(myfile, []) + uris
else:
for myfile in filedict:
filedict[myfile] += primaryuri_dict.get(myfile, [])
can_fetch=True
if listonly:
can_fetch = False
if can_fetch and not fetch_to_ro:
global _userpriv_test_write_file_cache
dirmode = 0o070
filemode = 0o60
modemask = 0o2
dir_gid = portage_gid
if "FAKED_MODE" in mysettings:
# When inside fakeroot, directories with portage's gid appear
# to have root's gid. Therefore, use root's gid instead of
# portage's gid to avoid spurrious permissions adjustments
# when inside fakeroot.
dir_gid = 0
distdir_dirs = [""]
try:
for x in distdir_dirs:
mydir = os.path.join(mysettings["DISTDIR"], x)
write_test_file = os.path.join(
mydir, ".__portage_test_write__")
try:
st = os.stat(mydir)
except OSError:
st = None
if st is not None and stat.S_ISDIR(st.st_mode):
if not (userfetch or userpriv):
continue
if _userpriv_test_write_file(mysettings, write_test_file):
continue
_userpriv_test_write_file_cache.pop(write_test_file, None)
if ensure_dirs(mydir, gid=dir_gid, mode=dirmode, mask=modemask):
if st is None:
# The directory has just been created
# and therefore it must be empty.
continue
writemsg(_("Adjusting permissions recursively: '%s'\n") % mydir,
noiselevel=-1)
def onerror(e):
raise # bail out on the first error that occurs during recursion
if not apply_recursive_permissions(mydir,
gid=dir_gid, dirmode=dirmode, dirmask=modemask,
filemode=filemode, filemask=modemask, onerror=onerror):
raise OperationNotPermitted(
_("Failed to apply recursive permissions for the portage group."))
except PortageException as e:
if not os.path.isdir(mysettings["DISTDIR"]):
writemsg("!!! %s\n" % str(e), noiselevel=-1)
writemsg(_("!!! Directory Not Found: DISTDIR='%s'\n") % mysettings["DISTDIR"], noiselevel=-1)
writemsg(_("!!! Fetching will fail!\n"), noiselevel=-1)
if can_fetch and \
not fetch_to_ro and \
not os.access(mysettings["DISTDIR"], os.W_OK):
writemsg(_("!!! No write access to '%s'\n") % mysettings["DISTDIR"],
noiselevel=-1)
can_fetch = False
distdir_writable = can_fetch and not fetch_to_ro
failed_files = set()
restrict_fetch_msg = False
for myfile in filedict:
"""
fetched status
0 nonexistent
1 partially downloaded
2 completely downloaded
"""
fetched = 0
orig_digests = mydigests.get(myfile, {})
if not (allow_missing_digests or listonly):
verifiable_hash_types = set(orig_digests).intersection(hashfunc_map)
verifiable_hash_types.discard("size")
if not verifiable_hash_types:
expected = set(hashfunc_map)
expected.discard("size")
expected = " ".join(sorted(expected))
got = set(orig_digests)
got.discard("size")
got = " ".join(sorted(got))
reason = (_("Insufficient data for checksum verification"),
got, expected)
writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile,
noiselevel=-1)
writemsg(_("!!! Reason: %s\n") % reason[0],
noiselevel=-1)
writemsg(_("!!! Got: %s\n!!! Expected: %s\n") % \
(reason[1], reason[2]), noiselevel=-1)
if fetchonly:
failed_files.add(myfile)
continue
else:
return 0
size = orig_digests.get("size")
if size == 0:
# Zero-byte distfiles are always invalid, so discard their digests.
del mydigests[myfile]
orig_digests.clear()
size = None
pruned_digests = orig_digests
if parallel_fetchonly:
pruned_digests = {}
if size is not None:
pruned_digests["size"] = size
myfile_path = os.path.join(mysettings["DISTDIR"], myfile)
has_space = True
has_space_superuser = True
file_lock = None
if listonly:
writemsg_stdout("\n", noiselevel=-1)
else:
# check if there is enough space in DISTDIR to completely store myfile
# overestimate the filesize so we aren't bitten by FS overhead
vfs_stat = None
if size is not None and hasattr(os, "statvfs"):
try:
vfs_stat = os.statvfs(mysettings["DISTDIR"])
except OSError as e:
writemsg_level("!!! statvfs('%s'): %s\n" %
(mysettings["DISTDIR"], e),
noiselevel=-1, level=logging.ERROR)
del e
if vfs_stat is not None:
try:
mysize = os.stat(myfile_path).st_size
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
mysize = 0
if (size - mysize + vfs_stat.f_bsize) >= \
(vfs_stat.f_bsize * vfs_stat.f_bavail):
if (size - mysize + vfs_stat.f_bsize) >= \
(vfs_stat.f_bsize * vfs_stat.f_bfree):
has_space_superuser = False
if not has_space_superuser:
has_space = False
elif secpass < 2:
has_space = False
elif userfetch:
has_space = False
if distdir_writable and use_locks:
lock_kwargs = {}
if fetchonly:
lock_kwargs["flags"] = os.O_NONBLOCK
try:
file_lock = lockfile(myfile_path,
wantnewlockfile=1, **lock_kwargs)
except TryAgain:
writemsg(_(">>> File '%s' is already locked by "
"another fetcher. Continuing...\n") % myfile,
noiselevel=-1)
continue
try:
if not listonly:
eout = EOutput()
eout.quiet = mysettings.get("PORTAGE_QUIET") == "1"
match, mystat = _check_distfile(
myfile_path, pruned_digests, eout, hash_filter=hash_filter)
if match:
# Skip permission adjustment for symlinks, since we don't
# want to modify anything outside of the primary DISTDIR,
# and symlinks typically point to PORTAGE_RO_DISTDIRS.
if distdir_writable and not os.path.islink(myfile_path):
try:
apply_secpass_permissions(myfile_path,
gid=portage_gid, mode=0o664, mask=0o2,
stat_cached=mystat)
except PortageException as e:
if not os.access(myfile_path, os.R_OK):
writemsg(_("!!! Failed to adjust permissions:"
" %s\n") % str(e), noiselevel=-1)
del e
continue
if distdir_writable and mystat is None:
# Remove broken symlinks if necessary.
try:
os.unlink(myfile_path)
except OSError:
pass
if mystat is not None:
if stat.S_ISDIR(mystat.st_mode):
writemsg_level(
_("!!! Unable to fetch file since "
"a directory is in the way: \n"
"!!! %s\n") % myfile_path,
level=logging.ERROR, noiselevel=-1)
return 0
if mystat.st_size == 0:
if distdir_writable:
try:
os.unlink(myfile_path)
except OSError:
pass
elif distdir_writable:
if mystat.st_size < fetch_resume_size and \
mystat.st_size < size:
# If the file already exists and the size does not
# match the existing digests, it may be that the
# user is attempting to update the digest. In this
# case, the digestgen() function will advise the
# user to use `ebuild --force foo.ebuild manifest`
# in order to force the old digests to be replaced.
# Since the user may want to keep this file, rename
# it instead of deleting it.
writemsg(_(">>> Renaming distfile with size "
"%d (smaller than " "PORTAGE_FETCH_RESU"
"ME_MIN_SIZE)\n") % mystat.st_size)
temp_filename = \
_checksum_failure_temp_file(
mysettings["DISTDIR"], myfile)
writemsg_stdout(_("Refetching... "
"File renamed to '%s'\n\n") % \
temp_filename, noiselevel=-1)
elif mystat.st_size >= size:
temp_filename = \
_checksum_failure_temp_file(
mysettings["DISTDIR"], myfile)
writemsg_stdout(_("Refetching... "
"File renamed to '%s'\n\n") % \
temp_filename, noiselevel=-1)
if distdir_writable and ro_distdirs:
readonly_file = None
for x in ro_distdirs:
filename = os.path.join(x, myfile)
match, mystat = _check_distfile(
filename, pruned_digests, eout, hash_filter=hash_filter)
if match:
readonly_file = filename
break
if readonly_file is not None:
try:
os.unlink(myfile_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
os.symlink(readonly_file, myfile_path)
continue
# this message is shown only after we know that
# the file is not already fetched
if not has_space:
writemsg(_("!!! Insufficient space to store %s in %s\n") % \
(myfile, mysettings["DISTDIR"]), noiselevel=-1)
if has_space_superuser:
writemsg(_("!!! Insufficient privileges to use "
"remaining space.\n"), noiselevel=-1)
if userfetch:
writemsg(_("!!! You may set FEATURES=\"-userfetch\""
" in /etc/make.conf in order to fetch with\n"
"!!! superuser privileges.\n"), noiselevel=-1)
if fsmirrors and not os.path.exists(myfile_path) and has_space:
for mydir in fsmirrors:
mirror_file = os.path.join(mydir, myfile)
try:
shutil.copyfile(mirror_file, myfile_path)
writemsg(_("Local mirror has file: %s\n") % myfile)
break
except (IOError, OSError) as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
try:
mystat = os.stat(myfile_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
else:
# Skip permission adjustment for symlinks, since we don't
# want to modify anything outside of the primary DISTDIR,
# and symlinks typically point to PORTAGE_RO_DISTDIRS.
if not os.path.islink(myfile_path):
try:
apply_secpass_permissions(myfile_path,
gid=portage_gid, mode=0o664, mask=0o2,
stat_cached=mystat)
except PortageException as e:
if not os.access(myfile_path, os.R_OK):
writemsg(_("!!! Failed to adjust permissions:"
" %s\n") % (e,), noiselevel=-1)
# If the file is empty then it's obviously invalid. Remove
# the empty file and try to download if possible.
if mystat.st_size == 0:
if distdir_writable:
try:
os.unlink(myfile_path)
except EnvironmentError:
pass
elif myfile not in mydigests:
# We don't have a digest, but the file exists. We must
# assume that it is fully downloaded.
continue
else:
if mystat.st_size < mydigests[myfile]["size"] and \
not restrict_fetch:
fetched = 1 # Try to resume this download.
elif parallel_fetchonly and \
mystat.st_size == mydigests[myfile]["size"]:
eout = EOutput()
eout.quiet = \
mysettings.get("PORTAGE_QUIET") == "1"
eout.ebegin(
"%s size ;-)" % (myfile, ))
eout.eend(0)
continue
else:
digests = _filter_unaccelarated_hashes(mydigests[myfile])
if hash_filter is not None:
digests = _apply_hash_filter(digests, hash_filter)
verified_ok, reason = verify_all(myfile_path, digests)
if not verified_ok:
writemsg(_("!!! Previously fetched"
" file: '%s'\n") % myfile, noiselevel=-1)
writemsg(_("!!! Reason: %s\n") % reason[0],
noiselevel=-1)
writemsg(_("!!! Got: %s\n"
"!!! Expected: %s\n") % \
(reason[1], reason[2]), noiselevel=-1)
if reason[0] == _("Insufficient data for checksum verification"):
return 0
if distdir_writable:
temp_filename = \
_checksum_failure_temp_file(
mysettings["DISTDIR"], myfile)
writemsg_stdout(_("Refetching... "
"File renamed to '%s'\n\n") % \
temp_filename, noiselevel=-1)
else:
eout = EOutput()
eout.quiet = \
mysettings.get("PORTAGE_QUIET", None) == "1"
if digests:
digests = list(digests)
digests.sort()
eout.ebegin(
"%s %s ;-)" % (myfile, " ".join(digests)))
eout.eend(0)
continue # fetch any remaining files
# Create a reversed list since that is optimal for list.pop().
uri_list = filedict[myfile][:]
uri_list.reverse()
checksum_failure_count = 0
tried_locations = set()
while uri_list:
loc = uri_list.pop()
# Eliminate duplicates here in case we've switched to
# "primaryuri" mode on the fly due to a checksum failure.
if loc in tried_locations:
continue
tried_locations.add(loc)
if listonly:
writemsg_stdout(loc+" ", noiselevel=-1)
continue
# allow different fetchcommands per protocol
protocol = loc[0:loc.find("://")]
global_config_path = GLOBAL_CONFIG_PATH
if mysettings['EPREFIX']:
global_config_path = os.path.join(mysettings['EPREFIX'],
GLOBAL_CONFIG_PATH.lstrip(os.sep))
missing_file_param = False
fetchcommand_var = "FETCHCOMMAND_" + protocol.upper()
fetchcommand = mysettings.get(fetchcommand_var)
if fetchcommand is None:
fetchcommand_var = "FETCHCOMMAND"
fetchcommand = mysettings.get(fetchcommand_var)
if fetchcommand is None:
writemsg_level(
_("!!! %s is unset. It should "
"have been defined in\n!!! %s/make.globals.\n") \
% (fetchcommand_var, global_config_path),
level=logging.ERROR, noiselevel=-1)
return 0
if "${FILE}" not in fetchcommand:
writemsg_level(
_("!!! %s does not contain the required ${FILE}"
" parameter.\n") % fetchcommand_var,
level=logging.ERROR, noiselevel=-1)
missing_file_param = True
resumecommand_var = "RESUMECOMMAND_" + protocol.upper()
resumecommand = mysettings.get(resumecommand_var)
if resumecommand is None:
resumecommand_var = "RESUMECOMMAND"
resumecommand = mysettings.get(resumecommand_var)
if resumecommand is None:
writemsg_level(
_("!!! %s is unset. It should "
"have been defined in\n!!! %s/make.globals.\n") \
% (resumecommand_var, global_config_path),
level=logging.ERROR, noiselevel=-1)
return 0
if "${FILE}" not in resumecommand:
writemsg_level(
_("!!! %s does not contain the required ${FILE}"
" parameter.\n") % resumecommand_var,
level=logging.ERROR, noiselevel=-1)
missing_file_param = True
if missing_file_param:
writemsg_level(
_("!!! Refer to the make.conf(5) man page for "
"information about how to\n!!! correctly specify "
"FETCHCOMMAND and RESUMECOMMAND.\n"),
level=logging.ERROR, noiselevel=-1)
if myfile != os.path.basename(loc):
return 0
if not can_fetch:
if fetched != 2:
try:
mysize = os.stat(myfile_path).st_size
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
mysize = 0
if mysize == 0:
writemsg(_("!!! File %s isn't fetched but unable to get it.\n") % myfile,
noiselevel=-1)
elif size is None or size > mysize:
writemsg(_("!!! File %s isn't fully fetched, but unable to complete it\n") % myfile,
noiselevel=-1)
else:
writemsg(_("!!! File %s is incorrect size, "
"but unable to retry.\n") % myfile, noiselevel=-1)
return 0
else:
continue
if fetched != 2 and has_space:
#we either need to resume or start the download
if fetched == 1:
try:
mystat = os.stat(myfile_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
fetched = 0
else:
if mystat.st_size < fetch_resume_size:
writemsg(_(">>> Deleting distfile with size "
"%d (smaller than " "PORTAGE_FETCH_RESU"
"ME_MIN_SIZE)\n") % mystat.st_size)
try:
os.unlink(myfile_path)
except OSError as e:
if e.errno not in \
(errno.ENOENT, errno.ESTALE):
raise
del e
fetched = 0
if fetched == 1:
#resume mode:
writemsg(_(">>> Resuming download...\n"))
locfetch=resumecommand
command_var = resumecommand_var
else:
#normal mode:
locfetch=fetchcommand
command_var = fetchcommand_var
writemsg_stdout(_(">>> Downloading '%s'\n") % \
_hide_url_passwd(loc))
variables = {
"DISTDIR": mysettings["DISTDIR"],
"URI": loc,
"FILE": myfile
}
myfetch = shlex_split(locfetch)
myfetch = [varexpand(x, mydict=variables) for x in myfetch]
myret = -1
try:
myret = _spawn_fetch(mysettings, myfetch)
finally:
try:
apply_secpass_permissions(myfile_path,
gid=portage_gid, mode=0o664, mask=0o2)
except FileNotFound:
pass
except PortageException as e:
if not os.access(myfile_path, os.R_OK):
writemsg(_("!!! Failed to adjust permissions:"
" %s\n") % str(e), noiselevel=-1)
del e
# If the file is empty then it's obviously invalid. Don't
# trust the return value from the fetcher. Remove the
# empty file and try to download again.
try:
if os.stat(myfile_path).st_size == 0:
os.unlink(myfile_path)
fetched = 0
continue
except EnvironmentError:
pass
if mydigests is not None and myfile in mydigests:
try:
mystat = os.stat(myfile_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
fetched = 0
else:
if stat.S_ISDIR(mystat.st_mode):
# This can happen if FETCHCOMMAND erroneously
# contains wget's -P option where it should
# instead have -O.
writemsg_level(
_("!!! The command specified in the "
"%s variable appears to have\n!!! "
"created a directory instead of a "
"normal file.\n") % command_var,
level=logging.ERROR, noiselevel=-1)
writemsg_level(
_("!!! Refer to the make.conf(5) "
"man page for information about how "
"to\n!!! correctly specify "
"FETCHCOMMAND and RESUMECOMMAND.\n"),
level=logging.ERROR, noiselevel=-1)
return 0
# no exception? file exists. let digestcheck() report
# an appropriately for size or checksum errors
# If the fetcher reported success and the file is
# too small, it's probably because the digest is
# bad (upstream changed the distfile). In this
# case we don't want to attempt to resume. Show a
# digest verification failure to that the user gets
# a clue about what just happened.
if myret != os.EX_OK and \
mystat.st_size < mydigests[myfile]["size"]:
# Fetch failed... Try the next one... Kill 404 files though.
if (mystat[stat.ST_SIZE]<100000) and (len(myfile)>4) and not ((myfile[-5:]==".html") or (myfile[-4:]==".htm")):
html404=re.compile("<title>.*(not found|404).*</title>",re.I|re.M)
with io.open(
_unicode_encode(myfile_path,
encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['content'], errors='replace'
) as f:
if html404.search(f.read()):
try:
os.unlink(mysettings["DISTDIR"]+"/"+myfile)
writemsg(_(">>> Deleting invalid distfile. (Improper 404 redirect from server.)\n"))
fetched = 0
continue
except (IOError, OSError):
pass
fetched = 1
continue
if True:
# File is the correct size--check the checksums for the fetched
# file NOW, for those users who don't have a stable/continuous
# net connection. This way we have a chance to try to download
# from another mirror...
digests = _filter_unaccelarated_hashes(mydigests[myfile])
if hash_filter is not None:
digests = _apply_hash_filter(digests, hash_filter)
verified_ok, reason = verify_all(myfile_path, digests)
if not verified_ok:
writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile,
noiselevel=-1)
writemsg(_("!!! Reason: %s\n") % reason[0],
noiselevel=-1)
writemsg(_("!!! Got: %s\n!!! Expected: %s\n") % \
(reason[1], reason[2]), noiselevel=-1)
if reason[0] == _("Insufficient data for checksum verification"):
return 0
temp_filename = \
_checksum_failure_temp_file(
mysettings["DISTDIR"], myfile)
writemsg_stdout(_("Refetching... "
"File renamed to '%s'\n\n") % \
temp_filename, noiselevel=-1)
fetched=0
checksum_failure_count += 1
if checksum_failure_count == \
checksum_failure_primaryuri:
# Switch to "primaryuri" mode in order
# to increase the probablility of
# of success.
primaryuris = \
primaryuri_dict.get(myfile)
if primaryuris:
uri_list.extend(
reversed(primaryuris))
if checksum_failure_count >= \
checksum_failure_max_tries:
break
else:
eout = EOutput()
eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1"
if digests:
eout.ebegin("%s %s ;-)" % \
(myfile, " ".join(sorted(digests))))
eout.eend(0)
fetched=2
break
else:
if not myret:
fetched=2
break
elif mydigests!=None:
writemsg(_("No digest file available and download failed.\n\n"),
noiselevel=-1)
finally:
if use_locks and file_lock:
unlockfile(file_lock)
file_lock = None
if listonly:
writemsg_stdout("\n", noiselevel=-1)
if fetched != 2:
if restrict_fetch and not restrict_fetch_msg:
restrict_fetch_msg = True
msg = _("\n!!! %s/%s"
" has fetch restriction turned on.\n"
"!!! This probably means that this "
"ebuild's files must be downloaded\n"
"!!! manually. See the comments in"
" the ebuild for more information.\n\n") % \
(mysettings["CATEGORY"], mysettings["PF"])
writemsg_level(msg,
level=logging.ERROR, noiselevel=-1)
elif restrict_fetch:
pass
elif listonly:
pass
elif not filedict[myfile]:
writemsg(_("Warning: No mirrors available for file"
" '%s'\n") % (myfile), noiselevel=-1)
else:
writemsg(_("!!! Couldn't download '%s'. Aborting.\n") % myfile,
noiselevel=-1)
if listonly:
failed_files.add(myfile)
continue
elif fetchonly:
failed_files.add(myfile)
continue
return 0
if failed_files:
return 0
return 1
def ExtractKernelVersion(base_dir):
"""
Try to figure out what kernel version we are running
@param base_dir: Path to sources (usually /usr/src/linux)
@type base_dir: string
@rtype: tuple( version[string], error[string])
@return:
1. tuple( version[string], error[string])
Either version or error is populated (but never both)
"""
lines = []
pathname = os.path.join(base_dir, 'Makefile')
try:
f = io.open(_unicode_encode(pathname,
encoding=_encodings['fs'], errors='strict'), mode='r',
encoding=_encodings['content'], errors='replace')
except OSError as details:
return (None, str(details))
except IOError as details:
return (None, str(details))
try:
for i in range(4):
lines.append(f.readline())
except OSError as details:
return (None, str(details))
except IOError as details:
return (None, str(details))
finally:
f.close()
lines = [l.strip() for l in lines]
version = ''
#XXX: The following code relies on the ordering of vars within the Makefile
for line in lines:
# split on the '=' then remove annoying whitespace
items = line.split("=")
items = [i.strip() for i in items]
if items[0] == 'VERSION' or \
items[0] == 'PATCHLEVEL':
version += items[1]
version += "."
elif items[0] == 'SUBLEVEL':
version += items[1]
elif items[0] == 'EXTRAVERSION' and \
items[-1] != items[0]:
version += items[1]
# Grab a list of files named localversion* and sort them
localversions = os.listdir(base_dir)
for x in range(len(localversions) - 1, -1, -1):
if localversions[x][:12] != "localversion":
del localversions[x]
localversions.sort()
# Append the contents of each to the version string, stripping ALL whitespace
for lv in localversions:
version += "".join(" ".join(grabfile(base_dir + "/" + lv)).split())
# Check the .config for a CONFIG_LOCALVERSION and append that too, also stripping whitespace
loader = KeyValuePairFileLoader(os.path.join(base_dir, ".config"), None)
kernelconfig, loader_errors = loader.load()
if loader_errors:
for file_path, file_errors in loader_errors.items():
for error_str in file_errors:
writemsg_level("%s: %s\n" % (file_path, error_str),
level=logging.ERROR, noiselevel=-1)
if kernelconfig and "CONFIG_LOCALVERSION" in kernelconfig:
version += "".join(shlex_split(kernelconfig["CONFIG_LOCALVERSION"]))
return (version, None)
def _add_repositories(portdir, portdir_overlay, prepos, ignored_map,
local_config, default_portdir):
"""Add overlays in PORTDIR_OVERLAY as repositories"""
overlays = []
portdir_orig = None
if portdir:
portdir = normalize_path(portdir)
portdir_orig = portdir
overlays.append(portdir)
try:
port_ov = [normalize_path(i) for i in shlex_split(portdir_overlay)]
except ValueError as e:
#File "/usr/lib/python3.2/shlex.py", line 168, in read_token
# raise ValueError("No closing quotation")
writemsg(_("!!! Invalid PORTDIR_OVERLAY:"
" %s: %s\n") % (e, portdir_overlay),
noiselevel=-1)
port_ov = []
overlays.extend(port_ov)
default_repo_opts = {}
if prepos['DEFAULT'].aliases is not None:
default_repo_opts['aliases'] = \
' '.join(prepos['DEFAULT'].aliases)
if prepos['DEFAULT'].eclass_overrides is not None:
default_repo_opts['eclass-overrides'] = \
' '.join(prepos['DEFAULT'].eclass_overrides)
if prepos['DEFAULT'].masters is not None:
default_repo_opts['masters'] = \
' '.join(prepos['DEFAULT'].masters)
if overlays:
# We need a copy of the original repos.conf data, since we're
# going to modify the prepos dict and some of the RepoConfig
# objects that we put in prepos may have to be discarded if
# they get overridden by a repository with the same name but
# a different location. This is common with repoman, for example,
# when temporarily overriding an rsync repo with another copy
# of the same repo from CVS.
repos_conf = prepos.copy()
#overlay priority is negative because we want them to be looked before any other repo
base_priority = 0
for ov in overlays:
# Ignore missing directory for 'gentoo' so that
# first sync with emerge-webrsync is possible.
if isdir_raise_eaccess(ov) or \
(base_priority == 0 and ov is portdir):
repo_opts = default_repo_opts.copy()
repo_opts['location'] = ov
repo = RepoConfig(None,
repo_opts,
local_config=local_config)
# repos_conf_opts contains options from repos.conf
repos_conf_opts = repos_conf.get(repo.name)
if repos_conf_opts is not None:
# Selectively copy only the attributes which
# repos.conf is allowed to override.
for k in ('aliases', 'auto_sync', 'eclass_overrides',
'force', 'masters', 'priority', 'sync_depth',
'sync_hooks_only_on_change', 'sync_type',
'sync_umask', 'sync_uri', 'sync_user',
'module_specific_options'):
v = getattr(repos_conf_opts, k, None)
if v is not None:
setattr(repo, k, v)
if repo.name in prepos:
# Silently ignore when PORTDIR overrides the location
# setting from the default repos.conf (bug #478544).
old_location = prepos[repo.name].location
if old_location is not None and \
old_location != repo.location and \
not (base_priority == 0 and
old_location == default_portdir):
ignored_map.setdefault(repo.name,
[]).append(old_location)
if old_location == portdir:
portdir = repo.location
if repo.priority is None:
if base_priority == 0 and ov == portdir_orig:
# If it's the original PORTDIR setting and it's not
# in PORTDIR_OVERLAY, then it will be assigned a
# special priority setting later.
pass
else:
repo.priority = base_priority
base_priority += 1
prepos[repo.name] = repo
else:
if not portage._sync_mode:
writemsg(
_("!!! Invalid PORTDIR_OVERLAY (not a dir): '%s'\n"
) % ov,
noiselevel=-1)
return portdir
def parse_args(argv, repoman_default_opts):
"""Use a customized optionParser to parse command line arguments for repoman
Args:
argv - a sequence of command line arguments
Returns:
(opts, args), just like a call to parser.parse_args()
"""
argv = portage._decode_argv(argv)
modes = {
"commit": "Run a scan then commit changes",
"ci": "Run a scan then commit changes",
"fix": "Fix simple QA issues (stray digests, missing digests)",
"full": "Scan directory tree and print all issues (not a summary)",
"help": "Show this screen",
"manifest": "Generate a Manifest (fetches files if necessary)",
"manifest-check": "Check Manifests for missing or incorrect digests",
"scan": "Scan directory tree for QA issues",
}
output_choices = {
"default": "The normal output format",
"column": "Columnar output suitable for use with grep",
}
mode_keys = list(modes)
mode_keys.sort()
output_keys = sorted(output_choices)
parser = argparse.ArgumentParser(
usage="repoman [options] [mode]",
description="Modes: %s" % " | ".join(mode_keys),
epilog="For more help consult the man page.",
)
parser.add_argument(
"-a",
"--ask",
dest="ask",
action="store_true",
default=False,
help="Request a confirmation before commiting",
)
parser.add_argument(
"-b",
"--bug",
dest="bug",
action="append",
metavar="<BUG-NO|BUG-URL>",
default=[],
help=(
"Mention a Gentoo or upstream bug in the commit footer; "
"takes either Gentoo bug number or full bug URL"
),
)
parser.add_argument(
"-c",
"--closes",
dest="closes",
action="append",
metavar="<PR-NO|PR-URL>",
default=[],
help=(
"Adds a Closes footer to close GitHub pull request (or compatible); "
"takes either GitHub PR number or full PR URL"
),
)
parser.add_argument(
"-m",
"--commitmsg",
dest="commitmsg",
help="specify a commit message on the command line",
)
parser.add_argument(
"-M",
"--commitmsgfile",
dest="commitmsgfile",
help="specify a path to a file that contains a commit message",
)
parser.add_argument(
"--digest",
choices=("y", "n"),
metavar="<y|n>",
help="Automatically update Manifest digests for modified files",
)
parser.add_argument(
"-p",
"--pretend",
dest="pretend",
default=False,
action="store_true",
help="don't commit or fix anything; just show what would be done",
)
parser.add_argument(
"-q",
"--quiet",
dest="quiet",
action="count",
default=0,
help="do not print unnecessary messages",
)
parser.add_argument(
"--echangelog",
choices=("y", "n", "force"),
metavar="<y|n|force>",
help=(
"for commit mode, call echangelog if ChangeLog is unmodified (or "
"regardless of modification if 'force' is specified)"
),
)
parser.add_argument(
"--experimental-inherit",
choices=("y", "n"),
metavar="<y|n>",
default="n",
help=(
"Enable experimental inherit.missing checks which may misbehave"
" when the internal eclass database becomes outdated"
),
)
parser.add_argument(
"--experimental-repository-modules",
choices=("y", "n"),
metavar="<y|n>",
default="n",
help="Enable experimental repository modules",
)
parser.add_argument(
"-f",
"--force",
dest="force",
action="store_true",
default=False,
help="Commit with QA violations",
)
parser.add_argument(
"-S",
"--straight-to-stable",
dest="straight_to_stable",
default=False,
action="store_true",
help="Allow committing straight to stable",
)
parser.add_argument(
"--vcs", dest="vcs", help="Force using specific VCS instead of autodetection"
)
parser.add_argument(
"-v",
"--verbose",
dest="verbosity",
action="count",
help="be very verbose in output",
default=0,
)
parser.add_argument(
"-V", "--version", dest="version", action="store_true", help="show version info"
)
parser.add_argument(
"-x",
"--xmlparse",
dest="xml_parse",
action="store_true",
default=False,
help="forces the metadata.xml parse check to be carried out",
)
parser.add_argument(
"--if-modified",
choices=("y", "n"),
default="n",
metavar="<y|n>",
help="only check packages that have uncommitted modifications",
)
parser.add_argument(
"-i",
"--ignore-arches",
dest="ignore_arches",
action="store_true",
default=False,
help="ignore arch-specific failures (where arch != host)",
)
parser.add_argument(
"--ignore-default-opts",
action="store_true",
help="do not use the REPOMAN_DEFAULT_OPTS environment variable",
)
parser.add_argument(
"-I",
"--ignore-masked",
dest="ignore_masked",
action="store_true",
default=False,
help="ignore masked packages (not allowed with commit mode)",
)
parser.add_argument(
"--include-arches",
dest="include_arches",
metavar="ARCHES",
action="append",
help=(
"A space separated list of arches used to "
"filter the selection of profiles for dependency checks"
),
)
parser.add_argument(
"--include-profiles",
dest="include_profiles",
metavar="PROFILES",
action="append",
help=(
"A space separated list of profiles used to "
"define the selection of profiles for dependency checks"
),
)
parser.add_argument(
"-d",
"--include-dev",
dest="include_dev",
action="store_true",
default=False,
help="include dev profiles in dependency checks",
)
parser.add_argument(
"-e",
"--include-exp-profiles",
choices=("y", "n"),
metavar="<y|n>",
default=False,
help="include exp profiles in dependency checks",
)
parser.add_argument(
"--unmatched-removal",
dest="unmatched_removal",
action="store_true",
default=False,
help=(
"enable strict checking of package.mask and package.unmask files"
" for unmatched removal atoms"
),
)
parser.add_argument(
"--without-mask",
dest="without_mask",
action="store_true",
default=False,
help=(
"behave as if no package.mask entries exist"
" (not allowed with commit mode)"
),
)
parser.add_argument(
"--output-style",
dest="output_style",
choices=output_keys,
help="select output type",
default="default",
)
parser.add_argument(
"-j",
"--jobs",
dest="jobs",
action="store",
type=int,
default=1,
help="Specifies the number of jobs (processes) to run simultaneously.",
)
parser.add_argument(
"-l",
"--load-average",
dest="load_average",
action="store",
type=float,
default=None,
help="Specifies that no new jobs (processes) should be started if there are others "
"jobs running and the load average is at least load (a floating-point number).",
)
parser.add_argument(
"--mode",
dest="mode",
choices=mode_keys,
help="specify which mode repoman will run in (default=full)",
)
# Modes help is included earlier, in the parser description.
parser.add_argument(
"mode_positional",
nargs="?",
metavar="mode",
choices=mode_keys,
help=argparse.SUPPRESS,
)
opts = parser.parse_args(argv[1:])
if not opts.ignore_default_opts:
default_opts = util.shlex_split(repoman_default_opts)
if default_opts:
opts = parser.parse_args(default_opts + argv[1:])
args = []
if opts.mode is not None:
args.append(opts.mode)
if opts.mode_positional is not None:
args.append(opts.mode_positional)
if len(set(args)) > 1:
parser.error("multiple modes specified: %s" % " ".join(args))
opts.mode = args[0] if args else None
if opts.mode == "help":
parser.print_help()
parser.exit()
if not opts.mode:
opts.mode = "full"
if opts.mode == "ci":
opts.mode = "commit" # backwards compat shortcut
# Use verbosity and quiet options to appropriately fiddle with the loglevel
for val in range(opts.verbosity):
logger = logging.getLogger()
logger.setLevel(logger.getEffectiveLevel() - 10)
for val in range(opts.quiet):
logger = logging.getLogger()
logger.setLevel(logger.getEffectiveLevel() + 10)
if opts.mode == "commit" and opts.commitmsg:
opts.commitmsg = _unicode_decode(opts.commitmsg)
if opts.mode == "commit" and not (opts.force or opts.pretend):
if opts.ignore_masked:
opts.ignore_masked = False
logging.warn("Commit mode automatically disables --ignore-masked")
if opts.without_mask:
opts.without_mask = False
logging.warn("Commit mode automatically disables --without-mask")
return (opts, args)
def _xpak_start(self):
tar_options = ""
if "xattr" in self.features:
process = subprocess.Popen(["tar", "--help"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = process.communicate()[0]
if b"--xattrs" in output:
tar_options = ["--xattrs", "--xattrs-include='*'"]
for x in portage.util.shlex_split(
self.env.get("PORTAGE_XATTR_EXCLUDE", "")):
tar_options.append(
portage._shell_quote("--xattrs-exclude=%s" % x))
tar_options = " ".join(tar_options)
decomp = _compressors.get(compression_probe(self.pkg_path))
if decomp is not None:
decomp_cmd = decomp.get("decompress")
elif tarfile.is_tarfile(
portage._unicode_encode(self.pkg_path,
encoding=portage._encodings["fs"],
errors="strict")):
decomp_cmd = "cat"
decomp = {
"compress": "cat",
"package": "sys-apps/coreutils",
}
else:
decomp_cmd = None
if decomp_cmd is None:
self.scheduler.output(
"!!! %s\n" % _("File compression header unrecognized: %s") %
self.pkg_path,
log_path=self.logfile,
background=self.background,
level=logging.ERROR,
)
self.returncode = 1
self._async_wait()
return
try:
decompression_binary = shlex_split(
varexpand(decomp_cmd, mydict=self.env))[0]
except IndexError:
decompression_binary = ""
if find_binary(decompression_binary) is None:
# Try alternative command if it exists
if decomp.get("decompress_alt"):
decomp_cmd = decomp.get("decompress_alt")
try:
decompression_binary = shlex_split(
varexpand(decomp_cmd, mydict=self.env))[0]
except IndexError:
decompression_binary = ""
if find_binary(decompression_binary) is None:
missing_package = decomp.get("package")
self.scheduler.output(
"!!! %s\n" %
_("File compression unsupported %s.\n Command was: %s.\n Maybe missing package: %s"
) % (
self.pkg_path,
varexpand(decomp_cmd, mydict=self.env),
missing_package,
),
log_path=self.logfile,
background=self.background,
level=logging.ERROR,
)
self.returncode = 1
self._async_wait()
return
pkg_xpak = portage.xpak.tbz2(self.pkg_path)
pkg_xpak.scan()
# SIGPIPE handling (128 + SIGPIPE) should be compatible with
# assert_sigpipe_ok() that's used by the ebuild unpack() helper.
self.args = [
self._shell_binary,
"-c",
("cmd0=(head -c %d -- %s) cmd1=(%s) cmd2=(tar -xp %s -C %s -f -); "
+ '"${cmd0[@]}" | "${cmd1[@]}" | "${cmd2[@]}"; ' +
"p=(${PIPESTATUS[@]}) ; for i in {0..2}; do " +
"if [[ ${p[$i]} != 0 && ${p[$i]} != %d ]] ; then " +
'echo command $(eval "echo \\"\'\\${cmd$i[*]}\'\\"") ' +
"failed with status ${p[$i]} ; exit ${p[$i]} ; fi ; done; " +
"if [ ${p[$i]} != 0 ] ; then " +
'echo command $(eval "echo \\"\'\\${cmd$i[*]}\'\\"") ' +
"failed with status ${p[$i]} ; exit ${p[$i]} ; fi ; " +
"exit 0 ;") % (
pkg_xpak.filestat.st_size - pkg_xpak.xpaksize,
portage._shell_quote(self.pkg_path),
decomp_cmd,
tar_options,
portage._shell_quote(self.image_dir),
128 + signal.SIGPIPE,
),
]
SpawnProcess._start(self)
def parse_args(argv, qahelp, repoman_default_opts):
"""Use a customized optionParser to parse command line arguments for repoman
Args:
argv - a sequence of command line arguments
qahelp - a dict of qa warning to help message
Returns:
(opts, args), just like a call to parser.parse_args()
"""
argv = portage._decode_argv(argv)
modes = {
"commit": "Run a scan then commit changes",
"ci": "Run a scan then commit changes",
"fix": "Fix simple QA issues (stray digests, missing digests)",
"full": "Scan directory tree and print all issues (not a summary)",
"help": "Show this screen",
"manifest": "Generate a Manifest (fetches files if necessary)",
"manifest-check": "Check Manifests for missing or incorrect digests",
"scan": "Scan directory tree for QA issues",
}
output_choices = {"default": "The normal output format", "column": "Columnar output suitable for use with grep"}
mode_keys = list(modes)
mode_keys.sort()
output_keys = sorted(output_choices)
parser = ArgumentParser(
usage="repoman [options] [mode]",
description="Modes: %s" % " | ".join(mode_keys),
epilog="For more help consult the man page.",
)
parser.add_argument(
"-a", "--ask", dest="ask", action="store_true", default=False, help="Request a confirmation before commiting"
)
parser.add_argument("-m", "--commitmsg", dest="commitmsg", help="specify a commit message on the command line")
parser.add_argument(
"-M", "--commitmsgfile", dest="commitmsgfile", help="specify a path to a file that contains a commit message"
)
parser.add_argument(
"--digest", choices=("y", "n"), metavar="<y|n>", help="Automatically update Manifest digests for modified files"
)
parser.add_argument(
"-p",
"--pretend",
dest="pretend",
default=False,
action="store_true",
help="don't commit or fix anything; just show what would be done",
)
parser.add_argument(
"-q", "--quiet", dest="quiet", action="count", default=0, help="do not print unnecessary messages"
)
parser.add_argument(
"--echangelog",
choices=("y", "n", "force"),
metavar="<y|n|force>",
help=(
"for commit mode, call echangelog if ChangeLog is unmodified (or "
"regardless of modification if 'force' is specified)"
),
)
parser.add_argument(
"--experimental-inherit",
choices=("y", "n"),
metavar="<y|n>",
default="n",
help=(
"Enable experimental inherit.missing checks which may misbehave"
" when the internal eclass database becomes outdated"
),
)
parser.add_argument(
"-f", "--force", dest="force", action="store_true", default=False, help="Commit with QA violations"
)
parser.add_argument(
"-S",
"--straight-to-stable",
dest="straight_to_stable",
default=False,
action="store_true",
help="Allow committing straight to stable",
)
parser.add_argument("--vcs", dest="vcs", help="Force using specific VCS instead of autodetection")
parser.add_argument(
"-v", "--verbose", dest="verbosity", action="count", help="be very verbose in output", default=0
)
parser.add_argument("-V", "--version", dest="version", action="store_true", help="show version info")
parser.add_argument(
"-x",
"--xmlparse",
dest="xml_parse",
action="store_true",
default=False,
help="forces the metadata.xml parse check to be carried out",
)
parser.add_argument(
"--if-modified",
choices=("y", "n"),
default="n",
metavar="<y|n>",
help="only check packages that have uncommitted modifications",
)
parser.add_argument(
"-i",
"--ignore-arches",
dest="ignore_arches",
action="store_true",
default=False,
help="ignore arch-specific failures (where arch != host)",
)
parser.add_argument(
"--ignore-default-opts", action="store_true", help="do not use the REPOMAN_DEFAULT_OPTS environment variable"
)
parser.add_argument(
"-I",
"--ignore-masked",
dest="ignore_masked",
action="store_true",
default=False,
help="ignore masked packages (not allowed with commit mode)",
)
parser.add_argument(
"--include-arches",
dest="include_arches",
metavar="ARCHES",
action="append",
help=("A space separated list of arches used to " "filter the selection of profiles for dependency checks"),
)
parser.add_argument(
"-d",
"--include-dev",
dest="include_dev",
action="store_true",
default=False,
help="include dev profiles in dependency checks",
)
parser.add_argument(
"-e",
"--include-exp-profiles",
choices=("y", "n"),
metavar="<y|n>",
default=False,
help="include exp profiles in dependency checks",
)
parser.add_argument(
"--unmatched-removal",
dest="unmatched_removal",
action="store_true",
default=False,
help=("enable strict checking of package.mask and package.unmask files" " for unmatched removal atoms"),
)
parser.add_argument(
"--without-mask",
dest="without_mask",
action="store_true",
default=False,
help=("behave as if no package.mask entries exist" " (not allowed with commit mode)"),
)
parser.add_argument(
"--output-style", dest="output_style", choices=output_keys, help="select output type", default="default"
)
parser.add_argument(
"--mode", dest="mode", choices=mode_keys, help="specify which mode repoman will run in (default=full)"
)
opts, args = parser.parse_known_args(argv[1:])
if not opts.ignore_default_opts:
default_opts = util.shlex_split(repoman_default_opts)
if default_opts:
opts, args = parser.parse_known_args(default_opts + sys.argv[1:])
if opts.mode == "help":
parser.print_help(short=False)
for arg in args:
if arg in modes:
if not opts.mode:
opts.mode = arg
break
else:
parser.error("invalid mode: %s" % arg)
if not opts.mode:
opts.mode = "full"
if opts.mode == "ci":
opts.mode = "commit" # backwards compat shortcut
# Use verbosity and quiet options to appropriately fiddle with the loglevel
for val in range(opts.verbosity):
logger = logging.getLogger()
logger.setLevel(logger.getEffectiveLevel() - 10)
for val in range(opts.quiet):
logger = logging.getLogger()
logger.setLevel(logger.getEffectiveLevel() + 10)
if opts.mode == "commit" and opts.commitmsg:
opts.commitmsg = _unicode_decode(opts.commitmsg)
if opts.mode == "commit" and not (opts.force or opts.pretend):
if opts.ignore_masked:
opts.ignore_masked = False
logging.warn("Commit mode automatically disables --ignore-masked")
if opts.without_mask:
opts.without_mask = False
logging.warn("Commit mode automatically disables --without-mask")
return (opts, args)
def _add_repositories(portdir, portdir_overlay, prepos, ignored_map, ignored_location_map):
"""Add overlays in PORTDIR_OVERLAY as repositories"""
overlays = []
if portdir:
portdir = normalize_path(portdir)
overlays.append(portdir)
try:
port_ov = [normalize_path(i) for i in shlex_split(portdir_overlay)]
except ValueError as e:
#File "/usr/lib/python3.2/shlex.py", line 168, in read_token
# raise ValueError("No closing quotation")
writemsg(_("!!! Invalid PORTDIR_OVERLAY:"
" %s: %s\n") % (e, portdir_overlay), noiselevel=-1)
port_ov = []
overlays.extend(port_ov)
default_repo_opts = {}
if prepos['DEFAULT'].aliases is not None:
default_repo_opts['aliases'] = \
' '.join(prepos['DEFAULT'].aliases)
if prepos['DEFAULT'].eclass_overrides is not None:
default_repo_opts['eclass-overrides'] = \
' '.join(prepos['DEFAULT'].eclass_overrides)
if prepos['DEFAULT'].masters is not None:
default_repo_opts['masters'] = \
' '.join(prepos['DEFAULT'].masters)
if overlays:
# We need a copy of the original repos.conf data, since we're
# going to modify the prepos dict and some of the RepoConfig
# objects that we put in prepos may have to be discarded if
# they get overridden by a repository with the same name but
# a different location. This is common with repoman, for example,
# when temporarily overriding an rsync repo with another copy
# of the same repo from CVS.
repos_conf = prepos.copy()
#overlay priority is negative because we want them to be looked before any other repo
base_priority = 0
for ov in overlays:
if os.path.isdir(ov):
repo_opts = default_repo_opts.copy()
repo_opts['location'] = ov
repo = RepoConfig(None, repo_opts)
# repos_conf_opts contains options from repos.conf
repos_conf_opts = repos_conf.get(repo.name)
if repos_conf_opts is not None:
# Selectively copy only the attributes which
# repos.conf is allowed to override.
for k in ('aliases', 'eclass_overrides', 'masters', 'priority'):
v = getattr(repos_conf_opts, k, None)
if v is not None:
setattr(repo, k, v)
if repo.name in prepos:
old_location = prepos[repo.name].location
if old_location is not None and old_location != repo.location:
ignored_map.setdefault(repo.name, []).append(old_location)
ignored_location_map[old_location] = repo.name
if old_location == portdir:
portdir = repo.user_location
if ov == portdir and portdir not in port_ov:
repo.priority = -1000
elif repo.priority is None:
repo.priority = base_priority
base_priority += 1
prepos[repo.name] = repo
else:
if not portage._sync_disabled_warnings:
writemsg(_("!!! Invalid PORTDIR_OVERLAY (not a dir): '%s'\n") % ov, noiselevel=-1)
return portdir
def _add_repositories(portdir, portdir_overlay, prepos, ignored_map, ignored_location_map):
"""Add overlays in PORTDIR_OVERLAY as repositories"""
overlays = []
if portdir:
portdir = normalize_path(portdir)
overlays.append(portdir)
try:
port_ov = [normalize_path(i) for i in shlex_split(portdir_overlay)]
except ValueError as e:
#File "/usr/lib/python3.2/shlex.py", line 168, in read_token
# raise ValueError("No closing quotation")
writemsg(_("!!! Invalid PORTDIR_OVERLAY:"
" %s: %s\n") % (e, portdir_overlay), noiselevel=-1)
port_ov = []
overlays.extend(port_ov)
default_repo_opts = {}
if prepos['DEFAULT'].aliases is not None:
default_repo_opts['aliases'] = \
' '.join(prepos['DEFAULT'].aliases)
if prepos['DEFAULT'].eclass_overrides is not None:
default_repo_opts['eclass-overrides'] = \
' '.join(prepos['DEFAULT'].eclass_overrides)
if prepos['DEFAULT'].masters is not None:
default_repo_opts['masters'] = \
' '.join(prepos['DEFAULT'].masters)
if overlays:
# We need a copy of the original repos.conf data, since we're
# going to modify the prepos dict and some of the RepoConfig
# objects that we put in prepos may have to be discarded if
# they get overridden by a repository with the same name but
# a different location. This is common with repoman, for example,
# when temporarily overriding an rsync repo with another copy
# of the same repo from CVS.
repos_conf = prepos.copy()
#overlay priority is negative because we want them to be looked before any other repo
base_priority = 0
for ov in overlays:
if os.path.isdir(ov):
repo_opts = default_repo_opts.copy()
repo_opts['location'] = ov
repo = RepoConfig(None, repo_opts)
# repos_conf_opts contains options from repos.conf
repos_conf_opts = repos_conf.get(repo.name)
if repos_conf_opts is not None:
# Selectively copy only the attributes which
# repos.conf is allowed to override.
for k in ('aliases', 'eclass_overrides', 'masters', 'priority'):
v = getattr(repos_conf_opts, k, None)
if v is not None:
setattr(repo, k, v)
if repo.name in prepos:
old_location = prepos[repo.name].location
if old_location is not None and old_location != repo.location:
ignored_map.setdefault(repo.name, []).append(old_location)
ignored_location_map[old_location] = repo.name
if old_location == portdir:
portdir = repo.user_location
if ov == portdir and portdir not in port_ov:
repo.priority = -1000
elif repo.priority is None:
repo.priority = base_priority
base_priority += 1
prepos[repo.name] = repo
else:
writemsg(_("!!! Invalid PORTDIR_OVERLAY"
" (not a dir): '%s'\n") % ov, noiselevel=-1)
return portdir
def parse_args(argv, repoman_default_opts):
"""Use a customized optionParser to parse command line arguments for repoman
Args:
argv - a sequence of command line arguments
Returns:
(opts, args), just like a call to parser.parse_args()
"""
argv = portage._decode_argv(argv)
modes = {
'commit': 'Run a scan then commit changes',
'ci': 'Run a scan then commit changes',
'fix': 'Fix simple QA issues (stray digests, missing digests)',
'full': 'Scan directory tree and print all issues (not a summary)',
'help': 'Show this screen',
'manifest': 'Generate a Manifest (fetches files if necessary)',
'manifest-check': 'Check Manifests for missing or incorrect digests',
'scan': 'Scan directory tree for QA issues'
}
output_choices = {
'default': 'The normal output format',
'column': 'Columnar output suitable for use with grep'
}
mode_keys = list(modes)
mode_keys.sort()
output_keys = sorted(output_choices)
parser = argparse.ArgumentParser(
usage="repoman [options] [mode]",
description="Modes: %s" % " | ".join(mode_keys),
epilog="For more help consult the man page.")
parser.add_argument('-a',
'--ask',
dest='ask',
action='store_true',
default=False,
help='Request a confirmation before commiting')
parser.add_argument(
'-b',
'--bug',
dest='bug',
action='append',
metavar='<BUG-NO|BUG-URL>',
default=[],
help=('Mention a Gentoo or upstream bug in the commit footer; '
'takes either Gentoo bug number or full bug URL'))
parser.add_argument(
'-c',
'--closes',
dest='closes',
action='append',
metavar='<PR-NO|PR-URL>',
default=[],
help=
('Adds a Closes footer to close GitHub pull request (or compatible); '
'takes either GitHub PR number or full PR URL'))
parser.add_argument(
'--copyright',
dest='copyright',
default=False,
action='store_true',
help=
('This option will update the copyright header to include the make.conf'
' COPYRIGHT_OWNER value if defined or the default "Gentoo Authors"'))
parser.add_argument('-m',
'--commitmsg',
dest='commitmsg',
help='specify a commit message on the command line')
parser.add_argument(
'-M',
'--commitmsgfile',
dest='commitmsgfile',
help='specify a path to a file that contains a commit message')
parser.add_argument(
'--digest',
choices=('y', 'n'),
metavar='<y|n>',
help='Automatically update Manifest digests for modified files')
parser.add_argument(
'-p',
'--pretend',
dest='pretend',
default=False,
action='store_true',
help='don\'t commit or fix anything; just show what would be done')
parser.add_argument('-q',
'--quiet',
dest="quiet",
action="count",
default=0,
help='do not print unnecessary messages')
parser.add_argument(
'--echangelog',
choices=('y', 'n', 'force'),
metavar="<y|n|force>",
help=(
'for commit mode, call echangelog if ChangeLog is unmodified (or '
'regardless of modification if \'force\' is specified)'))
parser.add_argument(
'--experimental-inherit',
choices=('y', 'n'),
metavar="<y|n>",
default='n',
help=('Enable experimental inherit.missing checks which may misbehave'
' when the internal eclass database becomes outdated'))
parser.add_argument('--experimental-repository-modules',
choices=('y', 'n'),
metavar="<y|n>",
default='n',
help='Enable experimental repository modules')
parser.add_argument('-f',
'--force',
dest='force',
action='store_true',
default=False,
help='Commit with QA violations')
parser.add_argument('-S',
'--straight-to-stable',
dest='straight_to_stable',
default=False,
action='store_true',
help='Allow committing straight to stable')
parser.add_argument(
'--vcs',
dest='vcs',
help='Force using specific VCS instead of autodetection')
parser.add_argument('-v',
'--verbose',
dest="verbosity",
action='count',
help='be very verbose in output',
default=0)
parser.add_argument('-V',
'--version',
dest='version',
action='store_true',
help='show version info')
parser.add_argument(
'-x',
'--xmlparse',
dest='xml_parse',
action='store_true',
default=False,
help='forces the metadata.xml parse check to be carried out')
parser.add_argument(
'--if-modified',
choices=('y', 'n'),
default='n',
metavar="<y|n>",
help='only check packages that have uncommitted modifications')
parser.add_argument(
'-i',
'--ignore-arches',
dest='ignore_arches',
action='store_true',
default=False,
help='ignore arch-specific failures (where arch != host)')
parser.add_argument(
"--ignore-default-opts",
action="store_true",
help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
parser.add_argument(
'-I',
'--ignore-masked',
dest='ignore_masked',
action='store_true',
default=False,
help='ignore masked packages (not allowed with commit mode)')
parser.add_argument(
'--include-arches',
dest='include_arches',
metavar='ARCHES',
action='append',
help=('A space separated list of arches used to '
'filter the selection of profiles for dependency checks'))
parser.add_argument('-d',
'--include-dev',
dest='include_dev',
action='store_true',
default=False,
help='include dev profiles in dependency checks')
parser.add_argument('-e',
'--include-exp-profiles',
choices=('y', 'n'),
metavar='<y|n>',
default=False,
help='include exp profiles in dependency checks')
parser.add_argument(
'--unmatched-removal',
dest='unmatched_removal',
action='store_true',
default=False,
help=('enable strict checking of package.mask and package.unmask files'
' for unmatched removal atoms'))
parser.add_argument('--without-mask',
dest='without_mask',
action='store_true',
default=False,
help=('behave as if no package.mask entries exist'
' (not allowed with commit mode)'))
parser.add_argument('--output-style',
dest='output_style',
choices=output_keys,
help='select output type',
default='default')
parser.add_argument(
'--mode',
dest='mode',
choices=mode_keys,
help='specify which mode repoman will run in (default=full)')
opts, args = parser.parse_known_args(argv[1:])
if not opts.ignore_default_opts:
default_opts = util.shlex_split(repoman_default_opts)
if default_opts:
opts, args = parser.parse_known_args(default_opts + sys.argv[1:])
if opts.mode == 'help':
parser.print_help(short=False)
for arg in args:
if arg in modes:
if not opts.mode:
opts.mode = arg
break
else:
parser.error("invalid mode: %s" % arg)
if not opts.mode:
opts.mode = 'full'
if opts.mode == 'ci':
opts.mode = 'commit' # backwards compat shortcut
# Use verbosity and quiet options to appropriately fiddle with the loglevel
for val in range(opts.verbosity):
logger = logging.getLogger()
logger.setLevel(logger.getEffectiveLevel() - 10)
for val in range(opts.quiet):
logger = logging.getLogger()
logger.setLevel(logger.getEffectiveLevel() + 10)
if opts.mode == 'commit' and opts.commitmsg:
opts.commitmsg = _unicode_decode(opts.commitmsg)
if opts.mode == 'commit' and not (opts.force or opts.pretend):
if opts.ignore_masked:
opts.ignore_masked = False
logging.warn('Commit mode automatically disables --ignore-masked')
if opts.without_mask:
opts.without_mask = False
logging.warn('Commit mode automatically disables --without-mask')
return (opts, args)
def _add_repositories(portdir, portdir_overlay, prepos,
ignored_map, local_config, default_portdir):
"""Add overlays in PORTDIR_OVERLAY as repositories"""
overlays = []
portdir_orig = None
if portdir:
portdir = normalize_path(portdir)
portdir_orig = portdir
overlays.append(portdir)
try:
port_ov = [normalize_path(i) for i in shlex_split(portdir_overlay)]
except ValueError as e:
#File "/usr/lib/python3.2/shlex.py", line 168, in read_token
# raise ValueError("No closing quotation")
writemsg(_("!!! Invalid PORTDIR_OVERLAY:"
" %s: %s\n") % (e, portdir_overlay), noiselevel=-1)
port_ov = []
overlays.extend(port_ov)
default_repo_opts = {}
if prepos['DEFAULT'].aliases is not None:
default_repo_opts['aliases'] = \
' '.join(prepos['DEFAULT'].aliases)
if prepos['DEFAULT'].eclass_overrides is not None:
default_repo_opts['eclass-overrides'] = \
' '.join(prepos['DEFAULT'].eclass_overrides)
if prepos['DEFAULT'].masters is not None:
default_repo_opts['masters'] = \
' '.join(prepos['DEFAULT'].masters)
if overlays:
# We need a copy of the original repos.conf data, since we're
# going to modify the prepos dict and some of the RepoConfig
# objects that we put in prepos may have to be discarded if
# they get overridden by a repository with the same name but
# a different location. This is common with repoman, for example,
# when temporarily overriding an rsync repo with another copy
# of the same repo from CVS.
repos_conf = prepos.copy()
#overlay priority is negative because we want them to be looked before any other repo
base_priority = 0
for ov in overlays:
# Ignore missing directory for 'gentoo' so that
# first sync with emerge-webrsync is possible.
if isdir_raise_eaccess(ov) or \
(base_priority == 0 and ov is portdir):
repo_opts = default_repo_opts.copy()
repo_opts['location'] = ov
repo = RepoConfig(None, repo_opts, local_config=local_config)
# repos_conf_opts contains options from repos.conf
repos_conf_opts = repos_conf.get(repo.name)
if repos_conf_opts is not None:
# Selectively copy only the attributes which
# repos.conf is allowed to override.
for k in ('aliases', 'auto_sync',
'clone_depth', 'eclass_overrides',
'force', 'masters', 'priority', 'strict_misc_digests',
'sync_depth', 'sync_hooks_only_on_change',
'sync_openpgp_key_path',
'sync_openpgp_key_refresh_retry_count',
'sync_openpgp_key_refresh_retry_delay_max',
'sync_openpgp_key_refresh_retry_delay_exp_base',
'sync_openpgp_key_refresh_retry_delay_mult',
'sync_openpgp_key_refresh_retry_overall_timeout',
'sync_type', 'sync_umask', 'sync_uri', 'sync_user',
'module_specific_options'):
v = getattr(repos_conf_opts, k, None)
if v is not None:
setattr(repo, k, v)
if repo.name in prepos:
# Silently ignore when PORTDIR overrides the location
# setting from the default repos.conf (bug #478544).
old_location = prepos[repo.name].location
if old_location is not None and \
old_location != repo.location and \
not (base_priority == 0 and
old_location == default_portdir):
ignored_map.setdefault(repo.name, []).append(old_location)
if old_location == portdir:
portdir = repo.location
if repo.priority is None:
if base_priority == 0 and ov == portdir_orig:
# If it's the original PORTDIR setting and it's not
# in PORTDIR_OVERLAY, then it will be assigned a
# special priority setting later.
pass
else:
repo.priority = base_priority
base_priority += 1
prepos[repo.name] = repo
else:
if not portage._sync_mode:
writemsg(_("!!! Invalid PORTDIR_OVERLAY (not a dir): '%s'\n") % ov, noiselevel=-1)
return portdir
def update(self):
''' Update existing git repository, and ignore the syncuri. We are
going to trust the user and assume that the user is in the branch
that he/she wants updated. We'll let the user manage branches with
git directly.
'''
if not self.has_bin:
return (1, False)
git_cmd_opts = ""
quiet = self.settings.get("PORTAGE_QUIET") == "1"
if self.repo.module_specific_options.get('sync-git-env'):
shlexed_env = shlex_split(
self.repo.module_specific_options['sync-git-env'])
env = dict(
(k, v) for k, _, v in (assignment.partition('=')
for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(env)
if self.repo.module_specific_options.get('sync-git-pull-env'):
shlexed_env = shlex_split(
self.repo.module_specific_options['sync-git-pull-env'])
pull_env = dict(
(k, v) for k, _, v in (assignment.partition('=')
for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(pull_env)
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
if self.repo.module_specific_options.get('sync-git-pull-extra-opts'):
git_cmd_opts += " %s" % self.repo.module_specific_options[
'sync-git-pull-extra-opts']
try:
remote_branch = portage._unicode_decode(
subprocess.check_output([
self.bin_command, 'rev-parse', '--abbrev-ref',
'--symbolic-full-name', '@{upstream}'
],
cwd=portage._unicode_encode(
self.repo.location))).rstrip('\n')
except subprocess.CalledProcessError as e:
msg = "!!! git rev-parse error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (e.returncode, False)
shallow = self.repo.sync_depth is not None and self.repo.sync_depth != 0
if shallow:
git_cmd_opts += " --depth %d" % self.repo.sync_depth
# For shallow fetch, unreachable objects may need to be pruned
# manually, in order to prevent automatic git gc calls from
# eventually failing (see bug 599008).
gc_cmd = ['git', '-c', 'gc.autodetach=false', 'gc', '--auto']
if quiet:
gc_cmd.append('--quiet')
exitcode = subprocess.call(gc_cmd,
cwd=portage._unicode_encode(
self.repo.location))
if exitcode != os.EX_OK:
msg = "!!! git gc error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
git_cmd = "%s fetch %s%s" % (
self.bin_command, remote_branch.partition('/')[0], git_cmd_opts)
writemsg_level(git_cmd + "\n")
rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
previous_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(
self.repo.location))
exitcode = portage.process.spawn_bash(
"cd %s ; exec %s" %
(portage._shell_quote(self.repo.location), git_cmd),
**self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! git fetch error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
if not self.verify_head(revision='refs/remotes/%s' % remote_branch):
return (1, False)
if shallow:
# Since the default merge strategy typically fails when
# the depth is not unlimited, `git reset --merge`.
merge_cmd = [self.bin_command, 'reset', '--merge']
else:
merge_cmd = [self.bin_command, 'merge']
merge_cmd.append('refs/remotes/%s' % remote_branch)
if quiet:
merge_cmd.append('--quiet')
exitcode = subprocess.call(merge_cmd,
cwd=portage._unicode_encode(
self.repo.location))
if exitcode != os.EX_OK:
msg = "!!! git merge error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
current_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(
self.repo.location))
return (os.EX_OK, current_rev != previous_rev)
def updated_config_files(count): self.assertEqual(count, sum(len(x[1]) for x in find_updated_config_files(eroot, shlex_split(config_protect))))
async def _async_test_simple(self, playground, metadata_xml_files, loop):
debug = playground.debug
settings = playground.settings
eprefix = settings["EPREFIX"]
eroot = settings["EROOT"]
trees = playground.trees
portdb = trees[eroot]["porttree"].dbapi
test_repo_location = settings.repositories["test_repo"].location
var_cache_edb = os.path.join(eprefix, "var", "cache", "edb")
cachedir = os.path.join(var_cache_edb, "dep")
cachedir_pregen = os.path.join(test_repo_location, "metadata",
"md5-cache")
portage_python = portage._python_interpreter
dispatch_conf_cmd = (
portage_python,
"-b",
"-Wd",
os.path.join(self.sbindir, "dispatch-conf"),
)
ebuild_cmd = (portage_python, "-b", "-Wd",
os.path.join(self.bindir, "ebuild"))
egencache_cmd = (
portage_python,
"-b",
"-Wd",
os.path.join(self.bindir, "egencache"),
"--repo",
"test_repo",
"--repositories-configuration",
settings.repositories.config_string(),
)
emerge_cmd = (portage_python, "-b", "-Wd",
os.path.join(self.bindir, "emerge"))
emaint_cmd = (portage_python, "-b", "-Wd",
os.path.join(self.sbindir, "emaint"))
env_update_cmd = (
portage_python,
"-b",
"-Wd",
os.path.join(self.sbindir, "env-update"),
)
etc_update_cmd = (BASH_BINARY, os.path.join(self.sbindir,
"etc-update"))
fixpackages_cmd = (
portage_python,
"-b",
"-Wd",
os.path.join(self.sbindir, "fixpackages"),
)
portageq_cmd = (
portage_python,
"-b",
"-Wd",
os.path.join(self.bindir, "portageq"),
)
quickpkg_cmd = (
portage_python,
"-b",
"-Wd",
os.path.join(self.bindir, "quickpkg"),
)
regenworld_cmd = (
portage_python,
"-b",
"-Wd",
os.path.join(self.sbindir, "regenworld"),
)
rm_binary = find_binary("rm")
self.assertEqual(rm_binary is None, False, "rm command not found")
rm_cmd = (rm_binary, )
egencache_extra_args = []
if self._have_python_xml():
egencache_extra_args.append("--update-use-local-desc")
test_ebuild = portdb.findname("dev-libs/A-1")
self.assertFalse(test_ebuild is None)
cross_prefix = os.path.join(eprefix, "cross_prefix")
cross_root = os.path.join(eprefix, "cross_root")
cross_eroot = os.path.join(cross_root, eprefix.lstrip(os.sep))
binhost_dir = os.path.join(eprefix, "binhost")
binhost_address = "127.0.0.1"
binhost_remote_path = "/binhost"
binhost_server = AsyncHTTPServer(
binhost_address, BinhostContentMap(binhost_remote_path,
binhost_dir), loop).__enter__()
binhost_uri = "http://{address}:{port}{path}".format(
address=binhost_address,
port=binhost_server.server_port,
path=binhost_remote_path,
)
binpkg_format = settings.get("BINPKG_FORMAT",
SUPPORTED_GENTOO_BINPKG_FORMATS[0])
self.assertIn(binpkg_format, ("xpak", "gpkg"))
if binpkg_format == "xpak":
foo_filename = "foo-0-1.xpak"
elif binpkg_format == "gpkg":
foo_filename = "foo-0-1.gpkg.tar"
test_commands = ()
if hasattr(argparse.ArgumentParser, "parse_intermixed_args"):
test_commands += (
emerge_cmd + ("--oneshot", "dev-libs/A", "-v", "dev-libs/A"), )
test_commands += (
emerge_cmd + (
"--usepkgonly",
"--root",
cross_root,
"--quickpkg-direct=y",
"--quickpkg-direct-root",
"/",
"dev-libs/A",
),
emerge_cmd + (
"--usepkgonly",
"--quickpkg-direct=y",
"--quickpkg-direct-root",
cross_root,
"dev-libs/A",
),
env_update_cmd,
portageq_cmd + (
"envvar",
"-v",
"CONFIG_PROTECT",
"EROOT",
"PORTAGE_CONFIGROOT",
"PORTAGE_TMPDIR",
"USERLAND",
),
etc_update_cmd,
dispatch_conf_cmd,
emerge_cmd + ("--version", ),
emerge_cmd + ("--info", ),
emerge_cmd + ("--info", "--verbose"),
emerge_cmd + ("--list-sets", ),
emerge_cmd + ("--check-news", ),
rm_cmd + ("-rf", cachedir),
rm_cmd + ("-rf", cachedir_pregen),
emerge_cmd + ("--regen", ),
rm_cmd + ("-rf", cachedir),
({
"FEATURES": "metadata-transfer"
}, ) + emerge_cmd + ("--regen", ),
rm_cmd + ("-rf", cachedir),
({
"FEATURES": "metadata-transfer"
}, ) + emerge_cmd + ("--regen", ),
rm_cmd + ("-rf", cachedir),
egencache_cmd + ("--update", ) + tuple(egencache_extra_args),
({
"FEATURES": "metadata-transfer"
}, ) + emerge_cmd + ("--metadata", ),
rm_cmd + ("-rf", cachedir),
({
"FEATURES": "metadata-transfer"
}, ) + emerge_cmd + ("--metadata", ),
emerge_cmd + ("--metadata", ),
rm_cmd + ("-rf", cachedir),
emerge_cmd + ("--oneshot", "virtual/foo"),
lambda: self.assertFalse(
os.path.exists(
os.path.join(pkgdir, "virtual", "foo", foo_filename))),
({
"FEATURES": "unmerge-backup"
}, ) + emerge_cmd + ("--unmerge", "virtual/foo"),
lambda: self.assertTrue(
os.path.exists(
os.path.join(pkgdir, "virtual", "foo", foo_filename))),
emerge_cmd + ("--pretend", "dev-libs/A"),
ebuild_cmd +
(test_ebuild, "manifest", "clean", "package", "merge"),
emerge_cmd +
("--pretend", "--tree", "--complete-graph", "dev-libs/A"),
emerge_cmd + ("-p", "dev-libs/B"),
emerge_cmd + ("-p", "--newrepo", "dev-libs/B"),
emerge_cmd + (
"-B",
"dev-libs/B",
),
emerge_cmd + (
"--oneshot",
"--usepkg",
"dev-libs/B",
),
# trigger clean prior to pkg_pretend as in bug #390711
ebuild_cmd + (test_ebuild, "unpack"),
emerge_cmd + (
"--oneshot",
"dev-libs/A",
),
emerge_cmd + (
"--noreplace",
"dev-libs/A",
),
emerge_cmd + (
"--config",
"dev-libs/A",
),
emerge_cmd + ("--info", "dev-libs/A", "dev-libs/B"),
emerge_cmd +
("--pretend", "--depclean", "--verbose", "dev-libs/B"),
emerge_cmd + (
"--pretend",
"--depclean",
),
emerge_cmd + ("--depclean", ),
quickpkg_cmd + (
"--include-config",
"y",
"dev-libs/A",
),
# Test bug #523684, where a file renamed or removed by the
# admin forces replacement files to be merged with config
# protection.
lambda: self.assertEqual(
0,
len(
list(
find_updated_config_files(
eroot, shlex_split(settings["CONFIG_PROTECT"])))),
),
lambda: os.unlink(os.path.join(eprefix, "etc", "A-0")),
emerge_cmd + ("--usepkgonly", "dev-libs/A"),
lambda: self.assertEqual(
1,
len(
list(
find_updated_config_files(
eroot, shlex_split(settings["CONFIG_PROTECT"])))),
),
emaint_cmd + ("--check", "all"),
emaint_cmd + ("--fix", "all"),
fixpackages_cmd,
regenworld_cmd,
portageq_cmd + ("match", eroot, "dev-libs/A"),
portageq_cmd + ("best_visible", eroot, "dev-libs/A"),
portageq_cmd + ("best_visible", eroot, "binary", "dev-libs/A"),
portageq_cmd + ("contents", eroot, "dev-libs/A-1"),
portageq_cmd + ("metadata", eroot, "ebuild", "dev-libs/A-1",
"EAPI", "IUSE", "RDEPEND"),
portageq_cmd + ("metadata", eroot, "binary", "dev-libs/A-1",
"EAPI", "USE", "RDEPEND"),
portageq_cmd + (
"metadata",
eroot,
"installed",
"dev-libs/A-1",
"EAPI",
"USE",
"RDEPEND",
),
portageq_cmd + ("owners", eroot, eroot + "usr"),
emerge_cmd + ("-p", eroot + "usr"),
emerge_cmd + ("-p", "--unmerge", "-q", eroot + "usr"),
emerge_cmd + ("--unmerge", "--quiet", "dev-libs/A"),
emerge_cmd + ("-C", "--quiet", "dev-libs/B"),
# If EMERGE_DEFAULT_OPTS contains --autounmask=n, then --autounmask
# must be specified with --autounmask-continue.
(
{
"EMERGE_DEFAULT_OPTS": "--autounmask=n"
}, ) + emerge_cmd + (
"--autounmask",
"--autounmask-continue",
"dev-libs/C",
),
# Verify that the above --autounmask-continue command caused
# USE=flag to be applied correctly to dev-libs/D.
portageq_cmd + ("match", eroot, "dev-libs/D[flag]"),
# Test cross-prefix usage, including chpathtool for binpkgs.
# EAPI 7
(
{
"EPREFIX": cross_prefix
}, ) + emerge_cmd + ("dev-libs/C", ),
({
"EPREFIX": cross_prefix
}, ) + portageq_cmd + ("has_version", cross_prefix, "dev-libs/C"),
({
"EPREFIX": cross_prefix
}, ) + portageq_cmd + ("has_version", cross_prefix, "dev-libs/D"),
({
"ROOT": cross_root
}, ) + emerge_cmd + ("dev-libs/D", ),
portageq_cmd + ("has_version", cross_eroot, "dev-libs/D"),
# EAPI 5
(
{
"EPREFIX": cross_prefix
}, ) + emerge_cmd + ("--usepkgonly", "dev-libs/A"),
({
"EPREFIX": cross_prefix
}, ) + portageq_cmd + ("has_version", cross_prefix, "dev-libs/A"),
({
"EPREFIX": cross_prefix
}, ) + portageq_cmd + ("has_version", cross_prefix, "dev-libs/B"),
({
"EPREFIX": cross_prefix
}, ) + emerge_cmd + ("-C", "--quiet", "dev-libs/B"),
({
"EPREFIX": cross_prefix
}, ) + emerge_cmd + ("-C", "--quiet", "dev-libs/A"),
({
"EPREFIX": cross_prefix
}, ) + emerge_cmd + ("dev-libs/A", ),
({
"EPREFIX": cross_prefix
}, ) + portageq_cmd + ("has_version", cross_prefix, "dev-libs/A"),
({
"EPREFIX": cross_prefix
}, ) + portageq_cmd + ("has_version", cross_prefix, "dev-libs/B"),
# Test ROOT support
(
{
"ROOT": cross_root
}, ) + emerge_cmd + ("dev-libs/B", ),
portageq_cmd + ("has_version", cross_eroot, "dev-libs/B"),
)
# Test binhost support if FETCHCOMMAND is available.
binrepos_conf_file = os.path.join(os.sep, eprefix, BINREPOS_CONF_FILE)
with open(binrepos_conf_file, "wt") as f:
f.write("[test-binhost]\n")
f.write("sync-uri = {}\n".format(binhost_uri))
fetchcommand = portage.util.shlex_split(
playground.settings["FETCHCOMMAND"])
fetch_bin = portage.process.find_binary(fetchcommand[0])
if fetch_bin is not None:
test_commands = test_commands + (
lambda: os.rename(pkgdir, binhost_dir),
emerge_cmd + ("-e", "--getbinpkgonly", "dev-libs/A"),
lambda: shutil.rmtree(pkgdir),
lambda: os.rename(binhost_dir, pkgdir),
# Remove binrepos.conf and test PORTAGE_BINHOST.
lambda: os.unlink(binrepos_conf_file),
lambda: os.rename(pkgdir, binhost_dir),
({
"PORTAGE_BINHOST": binhost_uri
}, ) + emerge_cmd + ("-fe", "--getbinpkgonly", "dev-libs/A"),
lambda: shutil.rmtree(pkgdir),
lambda: os.rename(binhost_dir, pkgdir),
)
distdir = playground.distdir
pkgdir = playground.pkgdir
fake_bin = os.path.join(eprefix, "bin")
portage_tmpdir = os.path.join(eprefix, "var", "tmp", "portage")
profile_path = settings.profile_path
user_config_dir = os.path.join(os.sep, eprefix, USER_CONFIG_PATH)
path = os.environ.get("PATH")
if path is not None and not path.strip():
path = None
if path is None:
path = ""
else:
path = ":" + path
path = fake_bin + path
pythonpath = os.environ.get("PYTHONPATH")
if pythonpath is not None and not pythonpath.strip():
pythonpath = None
if pythonpath is not None and pythonpath.split(
":")[0] == PORTAGE_PYM_PATH:
pass
else:
if pythonpath is None:
pythonpath = ""
else:
pythonpath = ":" + pythonpath
pythonpath = PORTAGE_PYM_PATH + pythonpath
env = {
"PORTAGE_OVERRIDE_EPREFIX":
eprefix,
"CLEAN_DELAY":
"0",
"DISTDIR":
distdir,
"EMERGE_WARNING_DELAY":
"0",
"INFODIR":
"",
"INFOPATH":
"",
"PATH":
path,
"PKGDIR":
pkgdir,
"PORTAGE_INST_GID":
str(portage.data.portage_gid),
"PORTAGE_INST_UID":
str(portage.data.portage_uid),
"PORTAGE_PYTHON":
portage_python,
"PORTAGE_REPOSITORIES":
settings.repositories.config_string(),
"PORTAGE_TMPDIR":
portage_tmpdir,
"PORTAGE_LOGDIR":
portage_tmpdir,
"PYTHONDONTWRITEBYTECODE":
os.environ.get("PYTHONDONTWRITEBYTECODE", ""),
"PYTHONPATH":
pythonpath,
"__PORTAGE_TEST_PATH_OVERRIDE":
fake_bin,
}
if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
env["__PORTAGE_TEST_HARDLINK_LOCKS"] = os.environ[
"__PORTAGE_TEST_HARDLINK_LOCKS"]
updates_dir = os.path.join(test_repo_location, "profiles", "updates")
dirs = [
cachedir,
cachedir_pregen,
cross_eroot,
cross_prefix,
distdir,
fake_bin,
portage_tmpdir,
updates_dir,
user_config_dir,
var_cache_edb,
]
etc_symlinks = ("dispatch-conf.conf", "etc-update.conf")
# Override things that may be unavailable, or may have portability
# issues when running tests in exotic environments.
# prepstrip - bug #447810 (bash read builtin EINTR problem)
true_symlinks = ["find", "prepstrip", "sed", "scanelf"]
true_binary = find_binary("true")
self.assertEqual(true_binary is None, False, "true command not found")
try:
for d in dirs:
ensure_dirs(d)
for x in true_symlinks:
os.symlink(true_binary, os.path.join(fake_bin, x))
for x in etc_symlinks:
os.symlink(os.path.join(self.cnf_etc_path, x),
os.path.join(eprefix, "etc", x))
with open(os.path.join(var_cache_edb, "counter"), "wb") as f:
f.write(b"100")
# non-empty system set keeps --depclean quiet
with open(os.path.join(profile_path, "packages"), "w") as f:
f.write("*dev-libs/token-system-pkg")
for cp, xml_data in metadata_xml_files:
with open(os.path.join(test_repo_location, cp, "metadata.xml"),
"w") as f:
f.write(playground.metadata_xml_template % xml_data)
with open(os.path.join(updates_dir, "1Q-2010"), "w") as f:
f.write("""
slotmove =app-doc/pms-3 2 3
move dev-util/git dev-vcs/git
""")
if debug:
# The subprocess inherits both stdout and stderr, for
# debugging purposes.
stdout = None
else:
# The subprocess inherits stderr so that any warnings
# triggered by python -Wd will be visible.
stdout = subprocess.PIPE
for args in test_commands:
if hasattr(args, "__call__"):
args()
continue
if isinstance(args[0], dict):
local_env = env.copy()
local_env.update(args[0])
args = args[1:]
else:
local_env = env
proc = await asyncio.create_subprocess_exec(*args,
env=local_env,
stderr=None,
stdout=stdout)
if debug:
await proc.wait()
else:
output, _err = await proc.communicate()
await proc.wait()
if proc.returncode != os.EX_OK:
portage.writemsg(output)
self.assertEqual(os.EX_OK, proc.returncode,
"emerge failed with args %s" % (args, ))
finally:
binhost_server.__exit__(None, None, None)
playground.cleanup()
def _add_overlays(portdir, portdir_overlay, prepos, ignored_map, ignored_location_map):
"""Add overlays in PORTDIR_OVERLAY as repositories"""
overlays = []
if portdir:
portdir = normalize_path(portdir)
overlays.append(portdir)
try:
port_ov = [normalize_path(i) for i in shlex_split(portdir_overlay)]
except ValueError as e:
#File "/usr/lib/python3.2/shlex.py", line 168, in read_token
# raise ValueError("No closing quotation")
writemsg(_("!!! Invalid PORTDIR_OVERLAY:"
" %s: %s\n") % (e, portdir_overlay), noiselevel=-1)
port_ov = []
overlays.extend(port_ov)
default_repo_opts = {}
if prepos['DEFAULT'].aliases is not None:
default_repo_opts['aliases'] = \
' '.join(prepos['DEFAULT'].aliases)
if prepos['DEFAULT'].eclass_overrides is not None:
default_repo_opts['eclass-overrides'] = \
' '.join(prepos['DEFAULT'].eclass_overrides)
if prepos['DEFAULT'].masters is not None:
default_repo_opts['masters'] = \
' '.join(prepos['DEFAULT'].masters)
if overlays:
#overlay priority is negative because we want them to be looked before any other repo
base_priority = 0
for ov in overlays:
if os.path.isdir(ov):
repo_opts = default_repo_opts.copy()
repo_opts['location'] = ov
repo = RepoConfig(None, repo_opts)
repo_conf_opts = prepos.get(repo.name)
if repo_conf_opts is not None:
if repo_conf_opts.aliases is not None:
repo_opts['aliases'] = \
' '.join(repo_conf_opts.aliases)
if repo_conf_opts.eclass_overrides is not None:
repo_opts['eclass-overrides'] = \
' '.join(repo_conf_opts.eclass_overrides)
if repo_conf_opts.masters is not None:
repo_opts['masters'] = \
' '.join(repo_conf_opts.masters)
repo = RepoConfig(repo.name, repo_opts)
if repo.name in prepos:
old_location = prepos[repo.name].location
if old_location is not None and old_location != repo.location:
ignored_map.setdefault(repo.name, []).append(old_location)
ignored_location_map[old_location] = repo.name
if old_location == portdir:
portdir = repo.user_location
prepos[repo.name].update(repo)
repo = prepos[repo.name]
else:
prepos[repo.name] = repo
if ov == portdir and portdir not in port_ov:
repo.priority = -1000
else:
repo.priority = base_priority
base_priority += 1
else:
writemsg(_("!!! Invalid PORTDIR_OVERLAY"
" (not a dir): '%s'\n") % ov, noiselevel=-1)
return portdir
def _start(self):
tar_options = ""
if "xattr" in self.features:
process = subprocess.Popen(["tar", "--help"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output = process.communicate()[0]
if b"--xattrs" in output:
tar_options = ["--xattrs", "--xattrs-include='*'"]
for x in portage.util.shlex_split(self.env.get("PORTAGE_XATTR_EXCLUDE", "")):
tar_options.append(portage._shell_quote("--xattrs-exclude=%s" % x))
tar_options = " ".join(tar_options)
decomp = _compressors.get(compression_probe(self.pkg_path))
if decomp is not None:
decomp_cmd = decomp.get("decompress")
else:
decomp_cmd = None
if decomp_cmd is None:
self.scheduler.output("!!! %s\n" %
_("File compression header unrecognized: %s") %
self.pkg_path, log_path=self.logfile,
background=self.background, level=logging.ERROR)
self.returncode = 1
self._async_wait()
return
try:
decompression_binary = shlex_split(varexpand(decomp_cmd, mydict=self.env))[0]
except IndexError:
decompression_binary = ""
if find_binary(decompression_binary) is None:
# Try alternative command if it exists
if _compressors.get(compression_probe(self.pkg_path)).get("decompress_alt"):
decomp_cmd = _compressors.get(
compression_probe(self.pkg_path)).get("decompress_alt")
try:
decompression_binary = shlex_split(varexpand(decomp_cmd, mydict=self.env))[0]
except IndexError:
decompression_binary = ""
if find_binary(decompression_binary) is None:
missing_package = _compressors.get(compression_probe(self.pkg_path)).get("package")
self.scheduler.output("!!! %s\n" %
_("File compression unsupported %s.\n Command was: %s.\n Maybe missing package: %s") %
(self.pkg_path, varexpand(decomp_cmd, mydict=self.env), missing_package), log_path=self.logfile,
background=self.background, level=logging.ERROR)
self.returncode = 1
self._async_wait()
return
pkg_xpak = portage.xpak.tbz2(self.pkg_path)
pkg_xpak.scan()
# SIGPIPE handling (128 + SIGPIPE) should be compatible with
# assert_sigpipe_ok() that's used by the ebuild unpack() helper.
self.args = [self._shell_binary, "-c",
("cmd0=(head -c %d -- %s) cmd1=(%s) cmd2=(tar -xp %s -C %s -f -); " + \
'"${cmd0[@]}" | "${cmd1[@]}" | "${cmd2[@]}"; ' + \
"p=(${PIPESTATUS[@]}) ; for i in {0..2}; do " + \
"if [[ ${p[$i]} != 0 && ${p[$i]} != %d ]] ; then " + \
"echo command $(eval \"echo \\\"'\\${cmd$i[*]}'\\\"\") " + \
"failed with status ${p[$i]} ; exit ${p[$i]} ; fi ; done; " + \
"if [ ${p[$i]} != 0 ] ; then " + \
"echo command $(eval \"echo \\\"'\\${cmd$i[*]}'\\\"\") " + \
"failed with status ${p[$i]} ; exit ${p[$i]} ; fi ; " + \
"exit 0 ;") % \
(pkg_xpak.filestat.st_size - pkg_xpak.xpaksize,
portage._shell_quote(self.pkg_path),
decomp_cmd,
tar_options,
portage._shell_quote(self.image_dir),
128 + signal.SIGPIPE)]
SpawnProcess._start(self)
def testSimple(self):
debug = False
install_something = """
S="${WORKDIR}"
pkg_pretend() {
einfo "called pkg_pretend for $CATEGORY/$PF"
}
src_install() {
einfo "installing something..."
insinto /usr/lib/${P}
echo "blah blah blah" > "${T}"/regular-file
doins "${T}"/regular-file
dosym regular-file /usr/lib/${P}/symlink || die
# Test CONFIG_PROTECT
insinto /etc
newins "${T}"/regular-file ${PN}-${SLOT%/*}
# Test code for bug #381629, using a copyright symbol encoded with latin-1.
# We use $(printf "\\xa9") rather than $'\\xa9', since printf apparently
# works in any case, while $'\\xa9' transforms to \\xef\\xbf\\xbd under
# some conditions. TODO: Find out why it transforms to \\xef\\xbf\\xbd when
# running tests for Python 3.2 (even though it's bash that is ultimately
# responsible for performing the transformation).
local latin_1_dir=/usr/lib/${P}/latin-1-$(printf "\\xa9")-directory
insinto "${latin_1_dir}"
echo "blah blah blah" > "${T}"/latin-1-$(printf "\\xa9")-regular-file || die
doins "${T}"/latin-1-$(printf "\\xa9")-regular-file
dosym latin-1-$(printf "\\xa9")-regular-file ${latin_1_dir}/latin-1-$(printf "\\xa9")-symlink || die
}
pkg_config() {
einfo "called pkg_config for $CATEGORY/$PF"
}
pkg_info() {
einfo "called pkg_info for $CATEGORY/$PF"
}
pkg_preinst() {
einfo "called pkg_preinst for $CATEGORY/$PF"
# Test that has_version and best_version work correctly with
# prefix (involves internal ROOT -> EROOT calculation in order
# to support ROOT override via the environment with EAPIs 3
# and later which support prefix).
if has_version $CATEGORY/$PN:$SLOT ; then
einfo "has_version detects an installed instance of $CATEGORY/$PN:$SLOT"
einfo "best_version reports that the installed instance is $(best_version $CATEGORY/$PN:$SLOT)"
else
einfo "has_version does not detect an installed instance of $CATEGORY/$PN:$SLOT"
fi
if [[ ${EPREFIX} != ${PORTAGE_OVERRIDE_EPREFIX} ]] ; then
if has_version --host-root $CATEGORY/$PN:$SLOT ; then
einfo "has_version --host-root detects an installed instance of $CATEGORY/$PN:$SLOT"
einfo "best_version --host-root reports that the installed instance is $(best_version $CATEGORY/$PN:$SLOT)"
else
einfo "has_version --host-root does not detect an installed instance of $CATEGORY/$PN:$SLOT"
fi
fi
}
"""
ebuilds = {
"dev-libs/A-1": {
"EAPI" : "5",
"IUSE" : "+flag",
"KEYWORDS": "x86",
"LICENSE": "GPL-2",
"MISC_CONTENT": install_something,
"RDEPEND": "flag? ( dev-libs/B[flag] )",
},
"dev-libs/B-1": {
"EAPI" : "5",
"IUSE" : "+flag",
"KEYWORDS": "x86",
"LICENSE": "GPL-2",
"MISC_CONTENT": install_something,
},
"dev-libs/C-1": {
"EAPI" : "6",
"KEYWORDS": "~x86",
"RDEPEND": "dev-libs/D[flag]",
},
"dev-libs/D-1": {
"EAPI" : "6",
"KEYWORDS": "~x86",
"IUSE" : "flag",
},
"virtual/foo-0": {
"EAPI" : "5",
"KEYWORDS": "x86",
"LICENSE": "GPL-2",
},
}
installed = {
"dev-libs/A-1": {
"EAPI" : "5",
"IUSE" : "+flag",
"KEYWORDS": "x86",
"LICENSE": "GPL-2",
"RDEPEND": "flag? ( dev-libs/B[flag] )",
"USE": "flag",
},
"dev-libs/B-1": {
"EAPI" : "5",
"IUSE" : "+flag",
"KEYWORDS": "x86",
"LICENSE": "GPL-2",
"USE": "flag",
},
"dev-libs/depclean-me-1": {
"EAPI" : "5",
"IUSE" : "",
"KEYWORDS": "x86",
"LICENSE": "GPL-2",
"USE": "",
},
"app-misc/depclean-me-1": {
"EAPI" : "5",
"IUSE" : "",
"KEYWORDS": "x86",
"LICENSE": "GPL-2",
"RDEPEND": "dev-libs/depclean-me",
"USE": "",
},
}
metadata_xml_files = (
(
"dev-libs/A",
{
"flags" : "<flag name='flag'>Description of how USE='flag' affects this package</flag>",
},
),
(
"dev-libs/B",
{
"flags" : "<flag name='flag'>Description of how USE='flag' affects this package</flag>",
},
),
)
playground = ResolverPlayground(
ebuilds=ebuilds, installed=installed, debug=debug)
settings = playground.settings
eprefix = settings["EPREFIX"]
eroot = settings["EROOT"]
trees = playground.trees
portdb = trees[eroot]["porttree"].dbapi
test_repo_location = settings.repositories["test_repo"].location
var_cache_edb = os.path.join(eprefix, "var", "cache", "edb")
cachedir = os.path.join(var_cache_edb, "dep")
cachedir_pregen = os.path.join(test_repo_location, "metadata", "md5-cache")
portage_python = portage._python_interpreter
dispatch_conf_cmd = (portage_python, "-b", "-Wd",
os.path.join(self.sbindir, "dispatch-conf"))
ebuild_cmd = (portage_python, "-b", "-Wd",
os.path.join(self.bindir, "ebuild"))
egencache_cmd = (portage_python, "-b", "-Wd",
os.path.join(self.bindir, "egencache"),
"--repo", "test_repo",
"--repositories-configuration", settings.repositories.config_string())
emerge_cmd = (portage_python, "-b", "-Wd",
os.path.join(self.bindir, "emerge"))
emaint_cmd = (portage_python, "-b", "-Wd",
os.path.join(self.sbindir, "emaint"))
env_update_cmd = (portage_python, "-b", "-Wd",
os.path.join(self.sbindir, "env-update"))
etc_update_cmd = (BASH_BINARY,
os.path.join(self.sbindir, "etc-update"))
fixpackages_cmd = (portage_python, "-b", "-Wd",
os.path.join(self.sbindir, "fixpackages"))
portageq_cmd = (portage_python, "-b", "-Wd",
os.path.join(self.bindir, "portageq"))
quickpkg_cmd = (portage_python, "-b", "-Wd",
os.path.join(self.bindir, "quickpkg"))
regenworld_cmd = (portage_python, "-b", "-Wd",
os.path.join(self.sbindir, "regenworld"))
rm_binary = find_binary("rm")
self.assertEqual(rm_binary is None, False,
"rm command not found")
rm_cmd = (rm_binary,)
egencache_extra_args = []
if self._have_python_xml():
egencache_extra_args.append("--update-use-local-desc")
test_ebuild = portdb.findname("dev-libs/A-1")
self.assertFalse(test_ebuild is None)
cross_prefix = os.path.join(eprefix, "cross_prefix")
cross_root = os.path.join(eprefix, "cross_root")
cross_eroot = os.path.join(cross_root, eprefix.lstrip(os.sep))
test_commands = (
env_update_cmd,
portageq_cmd + ("envvar", "-v", "CONFIG_PROTECT", "EROOT",
"PORTAGE_CONFIGROOT", "PORTAGE_TMPDIR", "USERLAND"),
etc_update_cmd,
dispatch_conf_cmd,
emerge_cmd + ("--version",),
emerge_cmd + ("--info",),
emerge_cmd + ("--info", "--verbose"),
emerge_cmd + ("--list-sets",),
emerge_cmd + ("--check-news",),
rm_cmd + ("-rf", cachedir),
rm_cmd + ("-rf", cachedir_pregen),
emerge_cmd + ("--regen",),
rm_cmd + ("-rf", cachedir),
({"FEATURES" : "metadata-transfer"},) + \
emerge_cmd + ("--regen",),
rm_cmd + ("-rf", cachedir),
({"FEATURES" : "metadata-transfer"},) + \
emerge_cmd + ("--regen",),
rm_cmd + ("-rf", cachedir),
egencache_cmd + ("--update",) + tuple(egencache_extra_args),
({"FEATURES" : "metadata-transfer"},) + \
emerge_cmd + ("--metadata",),
rm_cmd + ("-rf", cachedir),
({"FEATURES" : "metadata-transfer"},) + \
emerge_cmd + ("--metadata",),
emerge_cmd + ("--metadata",),
rm_cmd + ("-rf", cachedir),
emerge_cmd + ("--oneshot", "virtual/foo"),
lambda: self.assertFalse(os.path.exists(
os.path.join(pkgdir, "virtual", "foo-0.tbz2"))),
({"FEATURES" : "unmerge-backup"},) + \
emerge_cmd + ("--unmerge", "virtual/foo"),
lambda: self.assertTrue(os.path.exists(
os.path.join(pkgdir, "virtual", "foo-0.tbz2"))),
emerge_cmd + ("--pretend", "dev-libs/A"),
ebuild_cmd + (test_ebuild, "manifest", "clean", "package", "merge"),
emerge_cmd + ("--pretend", "--tree", "--complete-graph", "dev-libs/A"),
emerge_cmd + ("-p", "dev-libs/B"),
emerge_cmd + ("-p", "--newrepo", "dev-libs/B"),
emerge_cmd + ("-B", "dev-libs/B",),
emerge_cmd + ("--oneshot", "--usepkg", "dev-libs/B",),
# trigger clean prior to pkg_pretend as in bug #390711
ebuild_cmd + (test_ebuild, "unpack"),
emerge_cmd + ("--oneshot", "dev-libs/A",),
emerge_cmd + ("--noreplace", "dev-libs/A",),
emerge_cmd + ("--config", "dev-libs/A",),
emerge_cmd + ("--info", "dev-libs/A", "dev-libs/B"),
emerge_cmd + ("--pretend", "--depclean", "--verbose", "dev-libs/B"),
emerge_cmd + ("--pretend", "--depclean",),
emerge_cmd + ("--depclean",),
quickpkg_cmd + ("--include-config", "y", "dev-libs/A",),
# Test bug #523684, where a file renamed or removed by the
# admin forces replacement files to be merged with config
# protection.
lambda: self.assertEqual(0,
len(list(find_updated_config_files(eroot,
shlex_split(settings["CONFIG_PROTECT"]))))),
lambda: os.unlink(os.path.join(eprefix, "etc", "A-0")),
emerge_cmd + ("--usepkgonly", "dev-libs/A"),
lambda: self.assertEqual(1,
len(list(find_updated_config_files(eroot,
shlex_split(settings["CONFIG_PROTECT"]))))),
emaint_cmd + ("--check", "all"),
emaint_cmd + ("--fix", "all"),
fixpackages_cmd,
regenworld_cmd,
portageq_cmd + ("match", eroot, "dev-libs/A"),
portageq_cmd + ("best_visible", eroot, "dev-libs/A"),
portageq_cmd + ("best_visible", eroot, "binary", "dev-libs/A"),
portageq_cmd + ("contents", eroot, "dev-libs/A-1"),
portageq_cmd + ("metadata", eroot, "ebuild", "dev-libs/A-1", "EAPI", "IUSE", "RDEPEND"),
portageq_cmd + ("metadata", eroot, "binary", "dev-libs/A-1", "EAPI", "USE", "RDEPEND"),
portageq_cmd + ("metadata", eroot, "installed", "dev-libs/A-1", "EAPI", "USE", "RDEPEND"),
portageq_cmd + ("owners", eroot, eroot + "usr"),
emerge_cmd + ("-p", eroot + "usr"),
emerge_cmd + ("-p", "--unmerge", "-q", eroot + "usr"),
emerge_cmd + ("--unmerge", "--quiet", "dev-libs/A"),
emerge_cmd + ("-C", "--quiet", "dev-libs/B"),
emerge_cmd + ("--autounmask-continue", "dev-libs/C",),
# Verify that the above --autounmask-continue command caused
# USE=flag to be applied correctly to dev-libs/D.
portageq_cmd + ("match", eroot, "dev-libs/D[flag]"),
# Test cross-prefix usage, including chpathtool for binpkgs.
({"EPREFIX" : cross_prefix},) + \
emerge_cmd + ("--usepkgonly", "dev-libs/A"),
({"EPREFIX" : cross_prefix},) + \
portageq_cmd + ("has_version", cross_prefix, "dev-libs/A"),
({"EPREFIX" : cross_prefix},) + \
portageq_cmd + ("has_version", cross_prefix, "dev-libs/B"),
({"EPREFIX" : cross_prefix},) + \
emerge_cmd + ("-C", "--quiet", "dev-libs/B"),
({"EPREFIX" : cross_prefix},) + \
emerge_cmd + ("-C", "--quiet", "dev-libs/A"),
({"EPREFIX" : cross_prefix},) + \
emerge_cmd + ("dev-libs/A",),
({"EPREFIX" : cross_prefix},) + \
portageq_cmd + ("has_version", cross_prefix, "dev-libs/A"),
({"EPREFIX" : cross_prefix},) + \
portageq_cmd + ("has_version", cross_prefix, "dev-libs/B"),
# Test ROOT support
({"ROOT": cross_root},) + emerge_cmd + ("dev-libs/B",),
portageq_cmd + ("has_version", cross_eroot, "dev-libs/B"),
)
distdir = playground.distdir
pkgdir = playground.pkgdir
fake_bin = os.path.join(eprefix, "bin")
portage_tmpdir = os.path.join(eprefix, "var", "tmp", "portage")
profile_path = settings.profile_path
user_config_dir = os.path.join(os.sep, eprefix, USER_CONFIG_PATH)
path = os.environ.get("PATH")
if path is not None and not path.strip():
path = None
if path is None:
path = ""
else:
path = ":" + path
path = fake_bin + path
pythonpath = os.environ.get("PYTHONPATH")
if pythonpath is not None and not pythonpath.strip():
pythonpath = None
if pythonpath is not None and \
pythonpath.split(":")[0] == PORTAGE_PYM_PATH:
pass
else:
if pythonpath is None:
pythonpath = ""
else:
pythonpath = ":" + pythonpath
pythonpath = PORTAGE_PYM_PATH + pythonpath
env = {
"PORTAGE_OVERRIDE_EPREFIX" : eprefix,
"CLEAN_DELAY" : "0",
"DISTDIR" : distdir,
"EMERGE_WARNING_DELAY" : "0",
"INFODIR" : "",
"INFOPATH" : "",
"PATH" : path,
"PKGDIR" : pkgdir,
"PORTAGE_INST_GID" : str(portage.data.portage_gid),
"PORTAGE_INST_UID" : str(portage.data.portage_uid),
"PORTAGE_PYTHON" : portage_python,
"PORTAGE_REPOSITORIES" : settings.repositories.config_string(),
"PORTAGE_TMPDIR" : portage_tmpdir,
"PYTHONDONTWRITEBYTECODE" : os.environ.get("PYTHONDONTWRITEBYTECODE", ""),
"PYTHONPATH" : pythonpath,
"__PORTAGE_TEST_PATH_OVERRIDE" : fake_bin,
}
if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \
os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"]
updates_dir = os.path.join(test_repo_location, "profiles", "updates")
dirs = [cachedir, cachedir_pregen, cross_eroot, cross_prefix,
distdir, fake_bin, portage_tmpdir, updates_dir,
user_config_dir, var_cache_edb]
etc_symlinks = ("dispatch-conf.conf", "etc-update.conf")
# Override things that may be unavailable, or may have portability
# issues when running tests in exotic environments.
# prepstrip - bug #447810 (bash read builtin EINTR problem)
true_symlinks = ["find", "prepstrip", "sed", "scanelf"]
true_binary = find_binary("true")
self.assertEqual(true_binary is None, False,
"true command not found")
try:
for d in dirs:
ensure_dirs(d)
for x in true_symlinks:
os.symlink(true_binary, os.path.join(fake_bin, x))
for x in etc_symlinks:
os.symlink(os.path.join(self.cnf_etc_path, x),
os.path.join(eprefix, "etc", x))
with open(os.path.join(var_cache_edb, "counter"), 'wb') as f:
f.write(b"100")
# non-empty system set keeps --depclean quiet
with open(os.path.join(profile_path, "packages"), 'w') as f:
f.write("*dev-libs/token-system-pkg")
for cp, xml_data in metadata_xml_files:
with open(os.path.join(test_repo_location, cp, "metadata.xml"), 'w') as f:
f.write(playground.metadata_xml_template % xml_data)
with open(os.path.join(updates_dir, "1Q-2010"), 'w') as f:
f.write("""
slotmove =app-doc/pms-3 2 3
move dev-util/git dev-vcs/git
""")
if debug:
# The subprocess inherits both stdout and stderr, for
# debugging purposes.
stdout = None
else:
# The subprocess inherits stderr so that any warnings
# triggered by python -Wd will be visible.
stdout = subprocess.PIPE
for args in test_commands:
if hasattr(args, '__call__'):
args()
continue
if isinstance(args[0], dict):
local_env = env.copy()
local_env.update(args[0])
args = args[1:]
else:
local_env = env
proc = subprocess.Popen(args,
env=local_env, stdout=stdout)
if debug:
proc.wait()
else:
output = proc.stdout.readlines()
proc.wait()
proc.stdout.close()
if proc.returncode != os.EX_OK:
for line in output:
sys.stderr.write(_unicode_decode(line))
self.assertEqual(os.EX_OK, proc.returncode,
"emerge failed with args %s" % (args,))
finally:
playground.cleanup()
def parse_args(argv, repoman_default_opts):
"""Use a customized optionParser to parse command line arguments for repoman
Args:
argv - a sequence of command line arguments
Returns:
(opts, args), just like a call to parser.parse_args()
"""
argv = portage._decode_argv(argv)
modes = {
'commit': 'Run a scan then commit changes',
'ci': 'Run a scan then commit changes',
'fix': 'Fix simple QA issues (stray digests, missing digests)',
'full': 'Scan directory tree and print all issues (not a summary)',
'help': 'Show this screen',
'manifest': 'Generate a Manifest (fetches files if necessary)',
'manifest-check': 'Check Manifests for missing or incorrect digests',
'scan': 'Scan directory tree for QA issues'
}
output_choices = {
'default': 'The normal output format',
'column': 'Columnar output suitable for use with grep'
}
mode_keys = list(modes)
mode_keys.sort()
output_keys = sorted(output_choices)
parser = argparse.ArgumentParser(
usage="repoman [options] [mode]",
description="Modes: %s" % " | ".join(mode_keys),
epilog="For more help consult the man page.")
parser.add_argument(
'-a', '--ask', dest='ask', action='store_true',
default=False,
help='Request a confirmation before commiting')
parser.add_argument(
'-b', '--bug', dest='bug', action='append', metavar='<BUG-NO|BUG-URL>',
default=[],
help=(
'Mention a Gentoo or upstream bug in the commit footer; '
'takes either Gentoo bug number or full bug URL'))
parser.add_argument(
'-c', '--closes', dest='closes', action='append', metavar='<PR-NO|PR-URL>',
default=[],
help=(
'Adds a Closes footer to close GitHub pull request (or compatible); '
'takes either GitHub PR number or full PR URL'))
parser.add_argument(
'-m', '--commitmsg', dest='commitmsg',
help='specify a commit message on the command line')
parser.add_argument(
'-M', '--commitmsgfile', dest='commitmsgfile',
help='specify a path to a file that contains a commit message')
parser.add_argument(
'--digest', choices=('y', 'n'), metavar='<y|n>',
help='Automatically update Manifest digests for modified files')
parser.add_argument(
'-p', '--pretend', dest='pretend', default=False,
action='store_true',
help='don\'t commit or fix anything; just show what would be done')
parser.add_argument(
'-q', '--quiet', dest="quiet", action="count",
default=0,
help='do not print unnecessary messages')
parser.add_argument(
'--echangelog', choices=('y', 'n', 'force'), metavar="<y|n|force>",
help=(
'for commit mode, call echangelog if ChangeLog is unmodified (or '
'regardless of modification if \'force\' is specified)'))
parser.add_argument(
'--experimental-inherit', choices=('y', 'n'), metavar="<y|n>",
default='n',
help=(
'Enable experimental inherit.missing checks which may misbehave'
' when the internal eclass database becomes outdated'))
parser.add_argument(
'--experimental-repository-modules', choices=('y', 'n'), metavar="<y|n>",
default='n',
help='Enable experimental repository modules')
parser.add_argument(
'-f', '--force', dest='force', action='store_true',
default=False,
help='Commit with QA violations')
parser.add_argument(
'-S', '--straight-to-stable', dest='straight_to_stable',
default=False, action='store_true',
help='Allow committing straight to stable')
parser.add_argument(
'--vcs', dest='vcs',
help='Force using specific VCS instead of autodetection')
parser.add_argument(
'-v', '--verbose', dest="verbosity", action='count',
help='be very verbose in output', default=0)
parser.add_argument(
'-V', '--version', dest='version', action='store_true',
help='show version info')
parser.add_argument(
'-x', '--xmlparse', dest='xml_parse', action='store_true',
default=False,
help='forces the metadata.xml parse check to be carried out')
parser.add_argument(
'--if-modified', choices=('y', 'n'), default='n',
metavar="<y|n>",
help='only check packages that have uncommitted modifications')
parser.add_argument(
'-i', '--ignore-arches', dest='ignore_arches', action='store_true',
default=False,
help='ignore arch-specific failures (where arch != host)')
parser.add_argument(
"--ignore-default-opts",
action="store_true",
help="do not use the REPOMAN_DEFAULT_OPTS environment variable")
parser.add_argument(
'-I', '--ignore-masked', dest='ignore_masked', action='store_true',
default=False,
help='ignore masked packages (not allowed with commit mode)')
parser.add_argument(
'--include-arches',
dest='include_arches', metavar='ARCHES', action='append',
help=(
'A space separated list of arches used to '
'filter the selection of profiles for dependency checks'))
parser.add_argument(
'-d', '--include-dev', dest='include_dev', action='store_true',
default=False,
help='include dev profiles in dependency checks')
parser.add_argument(
'-e', '--include-exp-profiles', choices=('y', 'n'), metavar='<y|n>',
default=False,
help='include exp profiles in dependency checks')
parser.add_argument(
'--unmatched-removal', dest='unmatched_removal', action='store_true',
default=False,
help=(
'enable strict checking of package.mask and package.unmask files'
' for unmatched removal atoms'))
parser.add_argument(
'--without-mask', dest='without_mask', action='store_true',
default=False,
help=(
'behave as if no package.mask entries exist'
' (not allowed with commit mode)'))
parser.add_argument(
'--output-style', dest='output_style', choices=output_keys,
help='select output type', default='default')
parser.add_argument(
'--mode', dest='mode', choices=mode_keys,
help='specify which mode repoman will run in (default=full)')
opts, args = parser.parse_known_args(argv[1:])
if not opts.ignore_default_opts:
default_opts = util.shlex_split(repoman_default_opts)
if default_opts:
opts, args = parser.parse_known_args(default_opts + sys.argv[1:])
if opts.mode == 'help':
parser.print_help(short=False)
for arg in args:
if arg in modes:
if not opts.mode:
opts.mode = arg
break
else:
parser.error("invalid mode: %s" % arg)
if not opts.mode:
opts.mode = 'full'
if opts.mode == 'ci':
opts.mode = 'commit' # backwards compat shortcut
# Use verbosity and quiet options to appropriately fiddle with the loglevel
for val in range(opts.verbosity):
logger = logging.getLogger()
logger.setLevel(logger.getEffectiveLevel() - 10)
for val in range(opts.quiet):
logger = logging.getLogger()
logger.setLevel(logger.getEffectiveLevel() + 10)
if opts.mode == 'commit' and opts.commitmsg:
opts.commitmsg = _unicode_decode(opts.commitmsg)
if opts.mode == 'commit' and not (opts.force or opts.pretend):
if opts.ignore_masked:
opts.ignore_masked = False
logging.warn('Commit mode automatically disables --ignore-masked')
if opts.without_mask:
opts.without_mask = False
logging.warn('Commit mode automatically disables --without-mask')
return (opts, args)
def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True):
root = trees._running_eroot
mysettings = trees[root]["vartree"].settings
portdb = trees[root]["porttree"].dbapi
vardb = trees[root]["vartree"].dbapi
bindb = trees[root]["bintree"].dbapi
world_file = os.path.join(mysettings['EROOT'], WORLD_FILE)
world_list = grabfile(world_file)
world_modified = False
world_warnings = set()
updpath_map = {}
# Maps repo_name to list of updates. If a given repo has no updates
# directory, it will be omitted. If a repo has an updates directory
# but none need to be applied (according to timestamp logic), the
# value in the dict will be an empty list.
repo_map = {}
timestamps = {}
retupd = False
update_notice_printed = False
for repo_name in portdb.getRepositories():
repo = portdb.getRepositoryPath(repo_name)
updpath = os.path.join(repo, "profiles", "updates")
if not os.path.isdir(updpath):
continue
if updpath in updpath_map:
repo_map[repo_name] = updpath_map[updpath]
continue
try:
if if_mtime_changed:
update_data = grab_updates(updpath, prev_mtimes=prev_mtimes)
else:
update_data = grab_updates(updpath)
except DirectoryNotFound:
continue
myupd = []
updpath_map[updpath] = myupd
repo_map[repo_name] = myupd
if len(update_data) > 0:
for mykey, mystat, mycontent in update_data:
if not update_notice_printed:
update_notice_printed = True
writemsg_stdout("\n")
writemsg_stdout(
colorize("GOOD", _("Performing Global Updates\n")))
writemsg_stdout(
_("(Could take a couple of minutes if you have a lot of binary packages.)\n"
))
if not quiet:
writemsg_stdout(_(" %s='update pass' %s='binary update' "
"%s='/var/db update' %s='/var/db move'\n"
" %s='/var/db SLOT move' %s='binary move' "
"%s='binary SLOT move'\n %s='update /etc/portage/package.*'\n") % \
(bold("."), bold("*"), bold("#"), bold("@"), bold("s"), bold("%"), bold("S"), bold("p")))
valid_updates, errors = parse_updates(mycontent)
myupd.extend(valid_updates)
if not quiet:
writemsg_stdout(bold(mykey))
writemsg_stdout(len(valid_updates) * "." + "\n")
if len(errors) == 0:
# Update our internal mtime since we
# processed all of our directives.
timestamps[mykey] = mystat[stat.ST_MTIME]
else:
for msg in errors:
writemsg("%s\n" % msg, noiselevel=-1)
if myupd:
retupd = True
if retupd:
if os.access(bindb.bintree.pkgdir, os.W_OK):
# Call binarytree.populate(), since we want to make sure it's
# only populated with local packages here (getbinpkgs=0).
bindb.bintree.populate()
else:
bindb = None
master_repo = portdb.repositories.mainRepo()
if master_repo is not None:
master_repo = master_repo.name
if master_repo in repo_map:
repo_map['DEFAULT'] = repo_map[master_repo]
for repo_name, myupd in repo_map.items():
if repo_name == 'DEFAULT':
continue
if not myupd:
continue
def repo_match(repository):
return repository == repo_name or \
(repo_name == master_repo and repository not in repo_map)
def _world_repo_match(atoma, atomb):
"""
Check whether to perform a world change from atoma to atomb.
If best vardb match for atoma comes from the same repository
as the update file, allow that. Additionally, if portdb still
can find a match for old atom name, warn about that.
"""
matches = vardb.match(atoma)
if not matches:
matches = vardb.match(atomb)
if matches and \
repo_match(vardb.aux_get(best(matches), ['repository'])[0]):
if portdb.match(atoma):
world_warnings.add((atoma, atomb))
return True
else:
return False
for update_cmd in myupd:
for pos, atom in enumerate(world_list):
new_atom = update_dbentry(update_cmd, atom)
if atom != new_atom:
if _world_repo_match(atom, new_atom):
world_list[pos] = new_atom
world_modified = True
for update_cmd in myupd:
if update_cmd[0] == "move":
moves = vardb.move_ent(update_cmd, repo_match=repo_match)
if moves:
writemsg_stdout(moves * "@")
if bindb:
moves = bindb.move_ent(update_cmd, repo_match=repo_match)
if moves:
writemsg_stdout(moves * "%")
elif update_cmd[0] == "slotmove":
moves = vardb.move_slot_ent(update_cmd, repo_match=repo_match)
if moves:
writemsg_stdout(moves * "s")
if bindb:
moves = bindb.move_slot_ent(update_cmd,
repo_match=repo_match)
if moves:
writemsg_stdout(moves * "S")
if world_modified:
world_list.sort()
write_atomic(world_file, "".join("%s\n" % (x, ) for x in world_list))
if world_warnings:
# XXX: print warning that we've updated world entries
# and the old name still matches something (from an overlay)?
pass
if retupd:
def _config_repo_match(repo_name, atoma, atomb):
"""
Check whether to perform a world change from atoma to atomb.
If best vardb match for atoma comes from the same repository
as the update file, allow that. Additionally, if portdb still
can find a match for old atom name, warn about that.
"""
matches = vardb.match(atoma)
if not matches:
matches = vardb.match(atomb)
if not matches:
return False
repository = vardb.aux_get(best(matches), ['repository'])[0]
return repository == repo_name or \
(repo_name == master_repo and repository not in repo_map)
update_config_files(root,
shlex_split(mysettings.get("CONFIG_PROTECT", "")),
shlex_split(
mysettings.get("CONFIG_PROTECT_MASK", "")),
repo_map,
match_callback=_config_repo_match,
case_insensitive="case-insensitive-fs"
in mysettings.features)
# The above global updates proceed quickly, so they
# are considered a single mtimedb transaction.
if timestamps:
# We do not update the mtime in the mtimedb
# until after _all_ of the above updates have
# been processed because the mtimedb will
# automatically commit when killed by ctrl C.
for mykey, mtime in timestamps.items():
prev_mtimes[mykey] = mtime
do_upgrade_packagesmessage = False
# We gotta do the brute force updates for these now.
if True:
def onUpdate(_maxval, curval):
if curval > 0:
writemsg_stdout("#")
if quiet:
onUpdate = None
vardb.update_ents(repo_map, onUpdate=onUpdate)
if bindb:
def onUpdate(_maxval, curval):
if curval > 0:
writemsg_stdout("*")
if quiet:
onUpdate = None
bindb.update_ents(repo_map, onUpdate=onUpdate)
else:
do_upgrade_packagesmessage = 1
# Update progress above is indicated by characters written to stdout so
# we print a couple new lines here to separate the progress output from
# what follows.
writemsg_stdout("\n\n")
if do_upgrade_packagesmessage and bindb and \
bindb.cpv_all():
writemsg_stdout(
_(" ** Skipping packages. Run 'fixpackages' or set it in FEATURES to fix the tbz2's in the packages directory.\n"
))
writemsg_stdout(bold(_("Note: This can take a very long time.")))
writemsg_stdout("\n")
return retupd
def _start(self):
tar_options = ""
if "xattr" in self.features:
process = subprocess.Popen(["tar", "--help"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output = process.communicate()[0]
if b"--xattrs" in output:
tar_options = ["--xattrs", "--xattrs-include='*'"]
for x in portage.util.shlex_split(self.env.get("PORTAGE_XATTR_EXCLUDE", "")):
tar_options.append(portage._shell_quote("--xattrs-exclude=%s" % x))
tar_options = " ".join(tar_options)
decomp = _compressors.get(compression_probe(self.pkg_path))
if decomp is not None:
decomp_cmd = decomp.get("decompress")
else:
decomp_cmd = None
if decomp_cmd is None:
self.scheduler.output("!!! %s\n" %
_("File compression header unrecognized: %s") %
self.pkg_path, log_path=self.logfile,
background=self.background, level=logging.ERROR)
self.returncode = 1
self._async_wait()
return
try:
decompression_binary = shlex_split(varexpand(decomp_cmd, mydict=self.env))[0]
except IndexError:
decompression_binary = ""
if find_binary(decompression_binary) is None:
# Try alternative command if it exists
if _compressors.get(compression_probe(self.pkg_path)).get("decompress_alt"):
decomp_cmd = _compressors.get(
compression_probe(self.pkg_path)).get("decompress_alt")
try:
decompression_binary = shlex_split(varexpand(decomp_cmd, mydict=self.env))[0]
except IndexError:
decompression_binary = ""
if find_binary(decompression_binary) is None:
missing_package = _compressors.get(compression_probe(self.pkg_path)).get("package")
self.scheduler.output("!!! %s\n" %
_("File compression unsupported %s.\n Command was: %s.\n Maybe missing package: %s") %
(self.pkg_path, varexpand(decomp_cmd, mydict=self.env), missing_package), log_path=self.logfile,
background=self.background, level=logging.ERROR)
self.returncode = 1
self._async_wait()
return
# Add -q to decomp_cmd opts, in order to avoid "trailing garbage
# after EOF ignored" warning messages due to xpak trailer.
# SIGPIPE handling (128 + SIGPIPE) should be compatible with
# assert_sigpipe_ok() that's used by the ebuild unpack() helper.
self.args = [self._shell_binary, "-c",
("%s -cq -- %s | tar -xp %s -C %s -f - ; " + \
"p=(${PIPESTATUS[@]}) ; " + \
"if [[ ${p[0]} != 0 && ${p[0]} != %d ]] ; then " % (128 + signal.SIGPIPE) + \
"echo bzip2 failed with status ${p[0]} ; exit ${p[0]} ; fi ; " + \
"if [ ${p[1]} != 0 ] ; then " + \
"echo tar failed with status ${p[1]} ; exit ${p[1]} ; fi ; " + \
"exit 0 ;") % \
(decomp_cmd,
portage._shell_quote(self.pkg_path),
tar_options,
portage._shell_quote(self.image_dir))]
SpawnProcess._start(self)
