def testBashSyntax(self):
for parent, dirs, files in os.walk(PORTAGE_BIN_PATH):
parent = _unicode_decode(parent,
encoding=_encodings['fs'],
errors='strict')
for x in files:
x = _unicode_decode(x,
encoding=_encodings['fs'],
errors='strict')
ext = x.split('.')[-1]
if ext in ('.py', '.pyc', '.pyo'):
continue
x = os.path.join(parent, x)
st = os.lstat(x)
if not stat.S_ISREG(st.st_mode):
continue
# Check for bash shebang
f = open(
_unicode_encode(x,
encoding=_encodings['fs'],
errors='strict'), 'rb')
line = _unicode_decode(f.readline(),
encoding=_encodings['content'],
errors='replace')
f.close()
if line[:2] == '#!' and \
'bash' in line:
cmd = "%s -n %s" % (_shell_quote(BASH_BINARY),
_shell_quote(x))
status, output = subprocess_getstatusoutput(cmd)
self.assertEqual(os.WIFEXITED(status) and \
os.WEXITSTATUS(status) == os.EX_OK, True, msg=output)
def testBashSyntax(self):
for parent, dirs, files in os.walk(PORTAGE_BIN_PATH):
parent = _unicode_decode(parent,
encoding=_encodings['fs'], errors='strict')
for x in files:
x = _unicode_decode(x,
encoding=_encodings['fs'], errors='strict')
ext = x.split('.')[-1]
if ext in ('.py', '.pyc', '.pyo'):
continue
x = os.path.join(parent, x)
st = os.lstat(x)
if not stat.S_ISREG(st.st_mode):
continue
# Check for bash shebang
f = open(_unicode_encode(x,
encoding=_encodings['fs'], errors='strict'), 'rb')
line = _unicode_decode(f.readline(),
encoding=_encodings['content'], errors='replace')
f.close()
if line[:2] == '#!' and \
'bash' in line:
cmd = "%s -n %s" % (_shell_quote(BASH_BINARY), _shell_quote(x))
status, output = subprocess_getstatusoutput(cmd)
self.assertEqual(os.WIFEXITED(status) and \
os.WEXITSTATUS(status) == os.EX_OK, True, msg=output)
def new(self, **kwargs):
if kwargs:
self._kwargs(kwargs)
# initial checkout
msg = ">>> Starting initial cvs checkout with %s..." % self.repo.sync_uri
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
try:
os.rmdir(self.repo.location)
except OSError as e:
if e.errno != errno.ENOENT:
msg = "!!! existing '%s' directory; exiting." % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR)
return (1, False)
del e
cvs_root = self.repo.sync_uri
if (
portage.process.spawn_bash(
"cd %s; exec cvs -z0 -d %s co -P -d %s %s"
% (
portage._shell_quote(os.path.dirname(self.repo.location)),
portage._shell_quote(cvs_root),
portage._shell_quote(os.path.basename(self.repo.location)),
portage._shell_quote(self.repo.sync_cvs_repo),
),
**portage._native_kwargs(self.spawn_kwargs)
)
!= os.EX_OK
):
msg = "!!! cvs checkout error; exiting."
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR)
return (1, False)
return (0, False)
def new(self, **kwargs):
'''Do the initial clone of the repository'''
if kwargs:
self._kwargs(kwargs)
try:
if not os.path.exists(self.repo.location):
os.makedirs(self.repo.location)
self.logger(self.xterm_titles,
'Created new directory %s' % self.repo.location)
except IOError:
return (1, False)
sync_uri = self.repo.sync_uri
if sync_uri.startswith("file://"):
sync_uri = sync_uri[6:]
depth_arg = ''
if self.repo.sync_depth is not None:
depth_arg = '--depth %d ' % self.repo.sync_depth
git_cmd = "%s clone %s%s ." % (self.bin_command, depth_arg,
portage._shell_quote(sync_uri))
writemsg_level(git_cmd + "\n")
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), git_cmd),
**portage._native_kwargs(self.spawn_kwargs))
if exitcode != os.EX_OK:
msg = "!!! git clone error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
return (os.EX_OK, True)
def _start(self):
tar_options = ""
if "xattr" in self.features:
process = subprocess.Popen(["tar", "--help"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = process.communicate()[0]
if b"--xattrs" in output:
tar_options = "--xattrs"
# Add -q to bzip2 opts, in order to avoid "trailing garbage after
# EOF ignored" warning messages due to xpak trailer.
# SIGPIPE handling (128 + SIGPIPE) should be compatible with
# assert_sigpipe_ok() that's used by the ebuild unpack() helper.
self.args = [self._shell_binary, "-c",
("${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -cq -- %s | tar -xp %s -C %s -f - ; " + \
"p=(${PIPESTATUS[@]}) ; " + \
"if [[ ${p[0]} != 0 && ${p[0]} != %d ]] ; then " % (128 + signal.SIGPIPE) + \
"echo bzip2 failed with status ${p[0]} ; exit ${p[0]} ; fi ; " + \
"if [ ${p[1]} != 0 ] ; then " + \
"echo tar failed with status ${p[1]} ; exit ${p[1]} ; fi ; " + \
"exit 0 ;") % \
(portage._shell_quote(self.pkg_path),
tar_options,
portage._shell_quote(self.image_dir))]
SpawnProcess._start(self)
def new(self, **kwargs):
if kwargs:
self._kwargs(kwargs)
# initial checkout
cvs_root = self.repo.sync_uri
if (
portage.process.spawn_bash(
"cd %s; exec cvs -z0 -d %s co -P -d %s %s"
% (
portage._shell_quote(os.path.dirname(self.repo.location)),
portage._shell_quote(cvs_root),
portage._shell_quote(os.path.basename(self.repo.location)),
portage._shell_quote(
self.repo.module_specific_options["sync-cvs-repo"]
),
),
**self.spawn_kwargs
)
!= os.EX_OK
):
msg = "!!! cvs checkout error; exiting."
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR)
return (1, False)
return (0, False)
def new(self, **kwargs):
'''Do the initial clone of the repository'''
if kwargs:
self._kwargs(kwargs)
try:
if not os.path.exists(self.repo.location):
os.makedirs(self.repo.location)
self.logger(self.xterm_titles,
'Created new directory %s' % self.repo.location)
except IOError:
return (1, False)
sync_uri = self.repo.sync_uri
if sync_uri.startswith("file://"):
sync_uri = sync_uri[6:]
git_cmd_opts = ""
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
if self.repo.sync_depth is not None:
git_cmd_opts += " --depth %d" % self.repo.sync_depth
git_cmd = "%s clone%s %s ." % (self.bin_command, git_cmd_opts,
portage._shell_quote(sync_uri))
writemsg_level(git_cmd + "\n")
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), git_cmd),
**portage._native_kwargs(self.spawn_kwargs))
if exitcode != os.EX_OK:
msg = "!!! git clone error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
return (os.EX_OK, True)
def new(self, **kwargs):
'''Do the initial clone of the repository'''
if kwargs:
self._kwargs(kwargs)
emerge_config = self.options.get('emerge_config', None)
portdb = self.options.get('portdb', None)
try:
if not os.path.exists(self.repo.location):
os.makedirs(self.repo.location)
self.logger(self.xterm_titles,
'Created new directory %s' % self.repo.location)
except IOError:
return (1, False)
msg = ">>> Cloning git repository from upstream into %s..." % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
sync_uri = self.repo.sync_uri
if sync_uri.startswith("file://"):
sync_uri = sync_uri[6:]
exitcode = portage.process.spawn_bash("cd %s ; %s clone %s ." % \
(portage._shell_quote(self.repo.location),
self.bin_command,
portage._shell_quote(sync_uri)),
**portage._native_kwargs(self.spawn_kwargs))
if exitcode != os.EX_OK:
msg = "!!! git clone error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
msg = ">>> Git clone successful"
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
return (os.EX_OK, True)
def _start(self):
tar_options = ""
if "xattr" in self.features:
process = subprocess.Popen(["tar", "--help"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output = process.communicate()[0]
if b"--xattrs" in output:
tar_options = "--xattrs"
# Add -q to bzip2 opts, in order to avoid "trailing garbage after
# EOF ignored" warning messages due to xpak trailer.
# SIGPIPE handling (128 + SIGPIPE) should be compatible with
# assert_sigpipe_ok() that's used by the ebuild unpack() helper.
self.args = [self._shell_binary, "-c",
("${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -cq -- %s | tar -xp %s -C %s -f - ; " + \
"p=(${PIPESTATUS[@]}) ; " + \
"if [[ ${p[0]} != 0 && ${p[0]} != %d ]] ; then " % (128 + signal.SIGPIPE) + \
"echo bzip2 failed with status ${p[0]} ; exit ${p[0]} ; fi ; " + \
"if [ ${p[1]} != 0 ] ; then " + \
"echo tar failed with status ${p[1]} ; exit ${p[1]} ; fi ; " + \
"exit 0 ;") % \
(portage._shell_quote(self.pkg_path),
tar_options,
portage._shell_quote(self.image_dir))]
SpawnProcess._start(self)
def new(self, **kwargs):
'''Do the initial clone of the repository'''
if kwargs:
self._kwargs(kwargs)
if not self.has_bin:
return (1, False)
try:
if not os.path.exists(self.repo.location):
os.makedirs(self.repo.location)
self.logger(self.xterm_titles,
'Created new directory %s' % self.repo.location)
except IOError:
return (1, False)
sync_uri = self.repo.sync_uri
if sync_uri.startswith("file://"):
sync_uri = sync_uri[7:]
git_cmd_opts = ""
if self.repo.module_specific_options.get('sync-git-env'):
shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-env'])
env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(env)
if self.repo.module_specific_options.get('sync-git-clone-env'):
shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-clone-env'])
clone_env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(clone_env)
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
if self.repo.clone_depth is not None:
if self.repo.clone_depth != 0:
git_cmd_opts += " --depth %d" % self.repo.clone_depth
elif self.repo.sync_depth is not None:
if self.repo.sync_depth != 0:
git_cmd_opts += " --depth %d" % self.repo.sync_depth
else:
# default
git_cmd_opts += " --depth 1"
if self.repo.module_specific_options.get('sync-git-clone-extra-opts'):
git_cmd_opts += " %s" % self.repo.module_specific_options['sync-git-clone-extra-opts']
git_cmd = "%s clone%s %s ." % (self.bin_command, git_cmd_opts,
portage._shell_quote(sync_uri))
writemsg_level(git_cmd + "\n")
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), git_cmd),
**self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! git clone error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
if not self.verify_head():
return (1, False)
return (os.EX_OK, True)
def new(self, **kwargs):
'''Do the initial clone of the repository'''
if kwargs:
self._kwargs(kwargs)
if not self.has_bin:
return (1, False)
try:
if not os.path.exists(self.repo.location):
os.makedirs(self.repo.location)
self.logger(self.xterm_titles,
'Created new directory %s' % self.repo.location)
except IOError:
return (1, False)
sync_uri = self.repo.sync_uri
if sync_uri.startswith("file://"):
sync_uri = sync_uri[7:]
git_cmd_opts = ""
if self.repo.module_specific_options.get('sync-git-env'):
shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-env'])
env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(env)
if self.repo.module_specific_options.get('sync-git-clone-env'):
shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-clone-env'])
clone_env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(clone_env)
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
if self.repo.clone_depth is not None:
if self.repo.clone_depth != 0:
git_cmd_opts += " --depth %d" % self.repo.clone_depth
elif self.repo.sync_depth is not None:
if self.repo.sync_depth != 0:
git_cmd_opts += " --depth %d" % self.repo.sync_depth
else:
# default
git_cmd_opts += " --depth 1"
if self.repo.module_specific_options.get('sync-git-clone-extra-opts'):
git_cmd_opts += " %s" % self.repo.module_specific_options['sync-git-clone-extra-opts']
git_cmd = "%s clone%s %s ." % (self.bin_command, git_cmd_opts,
portage._shell_quote(sync_uri))
writemsg_level(git_cmd + "\n")
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), git_cmd),
**self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! git clone error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
if not self.verify_head():
return (1, False)
return (os.EX_OK, True)
def _start(self):
saved_env_path = self._get_saved_env_path()
dest_env_path = self._get_dest_env_path()
shell_cmd = "${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -c -- %s > %s" % \
(_shell_quote(saved_env_path),
_shell_quote(dest_env_path))
extractor_proc = SpawnProcess(
args=[BASH_BINARY, "-c", shell_cmd],
background=self.background,
env=self.settings.environ(),
scheduler=self.scheduler,
logfile=self.settings.get('PORTAGE_LOG_FILE'))
self._start_task(extractor_proc, self._extractor_exit)
def _start(self):
self.args = [self._shell_binary, "-c",
("bzip2 -dqc -- %s | tar -xp -C %s -f - ; " + \
"p=(${PIPESTATUS[@]}) ; " + \
"if [ ${p[0]} != 0 ] ; then " + \
"echo bzip2 failed with status ${p[0]} ; exit ${p[0]} ; fi ; " + \
"if [ ${p[1]} != 0 ] ; then " + \
"echo tar failed with status ${p[1]} ; exit ${p[1]} ; fi ; " + \
"exit 0 ;") % \
(portage._shell_quote(self.pkg_path),
portage._shell_quote(self.image_dir))]
self.env = os.environ.copy()
SpawnProcess._start(self)
def new(self, **kwargs):
if kwargs:
self._kwargs(kwargs)
#initial checkout
svn_root = self.repo.sync_uri
exitcode = portage.process.spawn_bash(
"cd %s; exec svn co %s ." % (portage._shell_quote(
self.repo.location), portage._shell_quote(svn_root)),
**self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! svn checkout error; exiting."
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR)
return (exitcode, False)
def _start(self):
self.args = [self._shell_binary, "-c",
("bzip2 -dqc -- %s | tar -xp -C %s -f - ; " + \
"p=(${PIPESTATUS[@]}) ; " + \
"if [ ${p[0]} != 0 ] ; then " + \
"echo bzip2 failed with status ${p[0]} ; exit ${p[0]} ; fi ; " + \
"if [ ${p[1]} != 0 ] ; then " + \
"echo tar failed with status ${p[1]} ; exit ${p[1]} ; fi ; " + \
"exit 0 ;") % \
(portage._shell_quote(self.pkg_path),
portage._shell_quote(self.image_dir))]
self.env = os.environ.copy()
SpawnProcess._start(self)
def _start(self):
saved_env_path = self._get_saved_env_path()
dest_env_path = self._get_dest_env_path()
shell_cmd = "${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -c -- %s > %s" % \
(_shell_quote(saved_env_path),
_shell_quote(dest_env_path))
extractor_proc = SpawnProcess(
args=[BASH_BINARY, "-c", shell_cmd],
background=self.background,
env=self.settings.environ(),
scheduler=self.scheduler,
logfile=self.settings.get('PORTAGE_LOG_FILE'))
self._start_task(extractor_proc, self._extractor_exit)
def new(self, **kwargs): if kwargs: self._kwargs(kwargs) #initial checkout svn_root = self.repo.sync_uri exitcode = portage.process.spawn_bash( "cd %s; exec svn co %s ." % (portage._shell_quote(self.repo.location), portage._shell_quote(svn_root)), **self.spawn_kwargs) if exitcode != os.EX_OK: msg = "!!! svn checkout error; exiting." self.logger(self.xterm_titles, msg) writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR) return (exitcode, False)
def update(self):
''' Update existing git repository, and ignore the syncuri. We are
going to trust the user and assume that the user is in the branch
that he/she wants updated. We'll let the user manage branches with
git directly.
'''
git_cmd_opts = ""
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
git_cmd = "%s pull%s" % (self.bin_command, git_cmd_opts)
writemsg_level(git_cmd + "\n")
rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
previous_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(self.repo.location))
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), git_cmd),
**portage._native_kwargs(self.spawn_kwargs))
if exitcode != os.EX_OK:
msg = "!!! git pull error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
current_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(self.repo.location))
return (os.EX_OK, current_rev != previous_rev)
def main(args):
opts, files = parse_args(args)
install_binary = Which('install', exclude=os.environ["__PORTAGE_HELPER_PATH"])
if install_binary is None:
sys.stderr.write("install: command not found\n")
return 127
cmdline = [install_binary]
cmdline += args
if sys.hexversion >= 0x3000000:
# We can't trust that the filesystem encoding (locale dependent)
# correctly matches the arguments, so use surrogateescape to
# pass through the original argv bytes for Python 3.
fs_encoding = sys.getfilesystemencoding()
cmdline = [x.encode(fs_encoding, 'surrogateescape') for x in cmdline]
files = [x.encode(fs_encoding, 'surrogateescape') for x in files]
if opts.target_directory is not None:
opts.target_directory = \
opts.target_directory.encode(fs_encoding, 'surrogateescape')
returncode = subprocess.call(cmdline)
if returncode == os.EX_OK:
returncode = copy_xattrs(opts, files)
if returncode != os.EX_OK:
portage.util.writemsg("!!! install: copy_xattrs failed with the "
"following arguments: %s\n" %
" ".join(portage._shell_quote(x) for x in args), noiselevel=-1)
return returncode
def main(args):
opts, files = parse_args(args)
install_binary = Which("install", exclude=os.environ["__PORTAGE_HELPER_PATH"])
if install_binary is None:
sys.stderr.write("install: command not found\n")
return 127
cmdline = [install_binary]
cmdline += args
# We can't trust that the filesystem encoding (locale dependent)
# correctly matches the arguments, so use surrogateescape to
# pass through the original argv bytes for Python 3.
fs_encoding = sys.getfilesystemencoding()
cmdline = [x.encode(fs_encoding, "surrogateescape") for x in cmdline]
files = [x.encode(fs_encoding, "surrogateescape") for x in files]
if opts.target_directory is not None:
opts.target_directory = opts.target_directory.encode(
fs_encoding, "surrogateescape"
)
returncode = subprocess.call(cmdline)
if returncode == os.EX_OK:
returncode = copy_xattrs(opts, files)
if returncode != os.EX_OK:
portage.util.writemsg(
"!!! install: copy_xattrs failed with the "
"following arguments: %s\n"
% " ".join(portage._shell_quote(x) for x in args),
noiselevel=-1,
)
return returncode
def _sync(self):
"""
Internal function to sync an existing CVS repository
@return: tuple of return code (0=success), whether the cache
needs to be updated
@rtype: (int, bool)
"""
cvs_root = self.repo.sync_uri
if cvs_root.startswith("cvs://"):
cvs_root = cvs_root[6:]
# cvs update
msg = ">>> Starting cvs update with %s..." % self.repo.sync_uri
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
exitcode = portage.process.spawn_bash(
"cd %s; exec cvs -z0 -q update -dP" % (portage._shell_quote(self.repo.location),),
**portage._native_kwargs(self.spawn_kwargs)
)
if exitcode != os.EX_OK:
msg = "!!! cvs update error; exiting."
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR)
return (exitcode, False)
def new(self, **kwargs): if kwargs: self._kwargs(kwargs) #initial checkout cvs_root = self.repo.sync_uri if portage.process.spawn_bash( "cd %s; exec cvs -z0 -d %s co -P -d %s %s" % (portage._shell_quote(os.path.dirname(self.repo.location)), portage._shell_quote(cvs_root), portage._shell_quote(os.path.basename(self.repo.location)), portage._shell_quote(self.repo.module_specific_options["sync-cvs-repo"])), **portage._native_kwargs(self.spawn_kwargs)) != os.EX_OK: msg = "!!! cvs checkout error; exiting." self.logger(self.xterm_titles, msg) writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR) return (1, False) return (0, False)
def update(self):
''' Update existing git repository, and ignore the syncuri. We are
going to trust the user and assume that the user is in the branch
that he/she wants updated. We'll let the user manage branches with
git directly.
'''
git_cmd_opts = ""
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
git_cmd = "%s pull%s" % (self.bin_command, git_cmd_opts)
writemsg_level(git_cmd + "\n")
rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
previous_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(
self.repo.location))
exitcode = portage.process.spawn_bash(
"cd %s ; exec %s" %
(portage._shell_quote(self.repo.location), git_cmd),
**portage._native_kwargs(self.spawn_kwargs))
if exitcode != os.EX_OK:
msg = "!!! git pull error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
current_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(
self.repo.location))
return (os.EX_OK, current_rev != previous_rev)
def _start(self):
# SIGPIPE handling (128 + SIGPIPE) should be compatible with
# assert_sigpipe_ok() that's used by the ebuild unpack() helper.
self.args = [self._shell_binary, "-c",
("bzip2 -dqc -- %s | tar -xp -C %s -f - ; " + \
"p=(${PIPESTATUS[@]}) ; " + \
"if [[ ${p[0]} != 0 && ${p[0]} != %d ]] ; then " % (128 + signal.SIGPIPE) + \
"echo bzip2 failed with status ${p[0]} ; exit ${p[0]} ; fi ; " + \
"if [ ${p[1]} != 0 ] ; then " + \
"echo tar failed with status ${p[1]} ; exit ${p[1]} ; fi ; " + \
"exit 0 ;") % \
(portage._shell_quote(self.pkg_path),
portage._shell_quote(self.image_dir))]
self.env = os.environ.copy()
SpawnProcess._start(self)
def _start(self):
# SIGPIPE handling (128 + SIGPIPE) should be compatible with
# assert_sigpipe_ok() that's used by the ebuild unpack() helper.
self.args = [self._shell_binary, "-c",
("bzip2 -dqc -- %s | tar -xp -C %s -f - ; " + \
"p=(${PIPESTATUS[@]}) ; " + \
"if [[ ${p[0]} != 0 && ${p[0]} != %d ]] ; then " % (128 + signal.SIGPIPE) + \
"echo bzip2 failed with status ${p[0]} ; exit ${p[0]} ; fi ; " + \
"if [ ${p[1]} != 0 ] ; then " + \
"echo tar failed with status ${p[1]} ; exit ${p[1]} ; fi ; " + \
"exit 0 ;") % \
(portage._shell_quote(self.pkg_path),
portage._shell_quote(self.image_dir))]
self.env = os.environ.copy()
SpawnProcess._start(self)
def _sync(self):
''' Update existing git repository, and ignore the syncuri. We are
going to trust the user and assume that the user is in the branch
that he/she wants updated. We'll let the user manage branches with
git directly.
'''
# No kwargs call here; this is internal, so it should have been
# called by something which set the internal variables
emerge_config = self.options.get('emerge_config', None)
portdb = self.options.get('portdb', None)
msg = ">>> Starting git pull in %s..." % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
exitcode = portage.process.spawn_bash("cd %s ; git pull" % \
(portage._shell_quote(self.repo.location),),
**portage._native_kwargs(self.spawn_kwargs))
if exitcode != os.EX_OK:
msg = "!!! git pull error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
msg = ">>> Git pull successful: %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
return (os.EX_OK, True)
def _start(self):
# Add -q to bzip2 opts, in order to avoid "trailing garbage after
# EOF ignored" warning messages due to xpak trailer.
# SIGPIPE handling (128 + SIGPIPE) should be compatible with
# assert_sigpipe_ok() that's used by the ebuild unpack() helper.
self.args = [self._shell_binary, "-c",
("${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -cq -- %s | tar -xp -C %s -f - ; " + \
"p=(${PIPESTATUS[@]}) ; " + \
"if [[ ${p[0]} != 0 && ${p[0]} != %d ]] ; then " % (128 + signal.SIGPIPE) + \
"echo bzip2 failed with status ${p[0]} ; exit ${p[0]} ; fi ; " + \
"if [ ${p[1]} != 0 ] ; then " + \
"echo tar failed with status ${p[1]} ; exit ${p[1]} ; fi ; " + \
"exit 0 ;") % \
(portage._shell_quote(self.pkg_path),
portage._shell_quote(self.image_dir))]
SpawnProcess._start(self)
def _start(self):
# Add -q to bzip2 opts, in order to avoid "trailing garbage after
# EOF ignored" warning messages due to xpak trailer.
# SIGPIPE handling (128 + SIGPIPE) should be compatible with
# assert_sigpipe_ok() that's used by the ebuild unpack() helper.
self.args = [self._shell_binary, "-c",
("${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -cq -- %s | tar -xp -C %s -f - ; " + \
"p=(${PIPESTATUS[@]}) ; " + \
"if [[ ${p[0]} != 0 && ${p[0]} != %d ]] ; then " % (128 + signal.SIGPIPE) + \
"echo bzip2 failed with status ${p[0]} ; exit ${p[0]} ; fi ; " + \
"if [ ${p[1]} != 0 ] ; then " + \
"echo tar failed with status ${p[1]} ; exit ${p[1]} ; fi ; " + \
"exit 0 ;") % \
(portage._shell_quote(self.pkg_path),
portage._shell_quote(self.image_dir))]
SpawnProcess._start(self)
def testDoebuildSpawn(self):
playground = ResolverPlayground()
try:
settings = config(clone=playground.settings)
cpv = 'sys-apps/portage-2.1'
metadata = {
'EAPI' : '2',
'INHERITED' : 'python eutils',
'IUSE' : 'build doc epydoc python3 selinux',
'LICENSE' : 'GPL-2',
'PROVIDE' : 'virtual/portage',
'RDEPEND' : '>=app-shells/bash-3.2_p17 >=dev-lang/python-2.6',
'SLOT' : '0',
}
root_config = playground.trees[playground.root]['root_config']
pkg = Package(built=False, cpv=cpv, installed=False,
metadata=metadata, root_config=root_config,
type_name='ebuild')
settings.setcpv(pkg)
settings['PORTAGE_PYTHON'] = _python_interpreter
settings['PORTAGE_BUILDDIR'] = os.path.join(
settings['PORTAGE_TMPDIR'], cpv)
settings['T'] = os.path.join(
settings['PORTAGE_BUILDDIR'], 'temp')
for x in ('PORTAGE_BUILDDIR', 'T'):
os.makedirs(settings[x])
# Create a fake environment, to pretend as if the ebuild
# has been sourced already.
open(os.path.join(settings['T'], 'environment'), 'wb')
task_scheduler = TaskScheduler()
for phase in ('_internal_test',):
# Test EbuildSpawnProcess by calling doebuild.spawn() with
# returnpid=False. This case is no longer used by portage
# internals since EbuildPhase is used instead and that passes
# returnpid=True to doebuild.spawn().
rval = doebuild_spawn("%s %s" % (_shell_quote(
os.path.join(settings["PORTAGE_BIN_PATH"],
os.path.basename(EBUILD_SH_BINARY))), phase),
settings, free=1)
self.assertEqual(rval, os.EX_OK)
ebuild_phase = EbuildPhase(background=False,
phase=phase, scheduler=task_scheduler.sched_iface,
settings=settings)
task_scheduler.add(ebuild_phase)
task_scheduler.run()
self.assertEqual(ebuild_phase.returncode, os.EX_OK)
ebuild_phase = MiscFunctionsProcess(background=False,
commands=['success_hooks'],
scheduler=task_scheduler.sched_iface, settings=settings)
task_scheduler.add(ebuild_phase)
task_scheduler.run()
self.assertEqual(ebuild_phase.returncode, os.EX_OK)
finally:
playground.cleanup()
def testDoebuildSpawn(self):
playground = ResolverPlayground()
try:
settings = config(clone=playground.settings)
cpv = 'sys-apps/portage-2.1'
metadata = {
'EAPI' : '2',
'INHERITED' : 'python eutils',
'IUSE' : 'build doc epydoc python3 selinux',
'LICENSE' : 'GPL-2',
'PROVIDE' : 'virtual/portage',
'RDEPEND' : '>=app-shells/bash-3.2_p17 >=dev-lang/python-2.6',
'SLOT' : '0',
}
root_config = playground.trees[playground.eroot]['root_config']
pkg = Package(built=False, cpv=cpv, installed=False,
metadata=metadata, root_config=root_config,
type_name='ebuild')
settings.setcpv(pkg)
settings['PORTAGE_PYTHON'] = _python_interpreter
settings['PORTAGE_BUILDDIR'] = os.path.join(
settings['PORTAGE_TMPDIR'], cpv)
settings['T'] = os.path.join(
settings['PORTAGE_BUILDDIR'], 'temp')
for x in ('PORTAGE_BUILDDIR', 'T'):
os.makedirs(settings[x])
# Create a fake environment, to pretend as if the ebuild
# has been sourced already.
open(os.path.join(settings['T'], 'environment'), 'wb').close()
scheduler = PollScheduler().sched_iface
for phase in ('_internal_test',):
# Test EbuildSpawnProcess by calling doebuild.spawn() with
# returnpid=False. This case is no longer used by portage
# internals since EbuildPhase is used instead and that passes
# returnpid=True to doebuild.spawn().
rval = doebuild_spawn("%s %s" % (_shell_quote(
os.path.join(settings["PORTAGE_BIN_PATH"],
os.path.basename(EBUILD_SH_BINARY))), phase),
settings, free=1)
self.assertEqual(rval, os.EX_OK)
ebuild_phase = EbuildPhase(background=False,
phase=phase, scheduler=scheduler,
settings=settings)
ebuild_phase.start()
ebuild_phase.wait()
self.assertEqual(ebuild_phase.returncode, os.EX_OK)
ebuild_phase = MiscFunctionsProcess(background=False,
commands=['success_hooks'],
scheduler=scheduler, settings=settings)
ebuild_phase.start()
ebuild_phase.wait()
self.assertEqual(ebuild_phase.returncode, os.EX_OK)
finally:
playground.cleanup()
def _start(self):
tar_options = ""
if "xattr" in self.features:
process = subprocess.Popen(["tar", "--help"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = process.communicate()[0]
if b"--xattrs" in output:
tar_options = ["--xattrs", "--xattrs-include='*'"]
for x in portage.util.shlex_split(
self.env.get("PORTAGE_XATTR_EXCLUDE", "")):
tar_options.append(
portage._shell_quote("--xattrs-exclude=%s" % x))
tar_options = " ".join(tar_options)
decomp_cmd = _decompressors.get(compression_probe(self.pkg_path))
if decomp_cmd is None:
self.scheduler.output(
"!!! %s\n" % _("File compression header unrecognized: %s") %
self.pkg_path,
log_path=self.logfile,
background=self.background,
level=logging.ERROR)
self.returncode = 1
self._async_wait()
return
# Add -q to decomp_cmd opts, in order to avoid "trailing garbage
# after EOF ignored" warning messages due to xpak trailer.
# SIGPIPE handling (128 + SIGPIPE) should be compatible with
# assert_sigpipe_ok() that's used by the ebuild unpack() helper.
self.args = [self._shell_binary, "-c",
("%s -cq -- %s | tar -xp %s -C %s -f - ; " + \
"p=(${PIPESTATUS[@]}) ; " + \
"if [[ ${p[0]} != 0 && ${p[0]} != %d ]] ; then " % (128 + signal.SIGPIPE) + \
"echo bzip2 failed with status ${p[0]} ; exit ${p[0]} ; fi ; " + \
"if [ ${p[1]} != 0 ] ; then " + \
"echo tar failed with status ${p[1]} ; exit ${p[1]} ; fi ; " + \
"exit 0 ;") % \
(decomp_cmd,
portage._shell_quote(self.pkg_path),
tar_options,
portage._shell_quote(self.image_dir))]
SpawnProcess._start(self)
def new(self, **kwargs):
"""Do the initial clone of the repository"""
if kwargs:
self._kwargs(kwargs)
try:
if not os.path.exists(self.repo.location):
os.makedirs(self.repo.location)
self.logger(self.xterm_titles,
"Created new directory %s" % self.repo.location)
except IOError:
return (1, False)
sync_uri = self.repo.sync_uri
if sync_uri.startswith("file://"):
sync_uri = sync_uri[7:]
hg_cmd_opts = ""
if self.repo.module_specific_options.get("sync-mercurial-env"):
shlexed_env = shlex_split(
self.repo.module_specific_options["sync-mercurial-env"])
env = dict(
(k, v) for k, _, v in (assignment.partition("=")
for assignment in shlexed_env) if k)
self.spawn_kwargs["env"].update(env)
if self.repo.module_specific_options.get("sync-mercurial-clone-env"):
shlexed_env = shlex_split(
self.repo.module_specific_options["sync-mercurial-clone-env"])
clone_env = dict(
(k, v) for k, _, v in (assignment.partition("=")
for assignment in shlexed_env) if k)
self.spawn_kwargs["env"].update(clone_env)
if self.settings.get("PORTAGE_QUIET") == "1":
hg_cmd_opts += " --quiet"
if self.repo.module_specific_options.get(
"sync-mercurial-clone-extra-opts"):
hg_cmd_opts += (
" %s" % self.repo.
module_specific_options["sync-mercurial-clone-extra-opts"])
hg_cmd = "%s clone%s %s ." % (
self.bin_command,
hg_cmd_opts,
portage._shell_quote(sync_uri),
)
writemsg_level(hg_cmd + "\n")
exitcode = portage.process.spawn(shlex_split(hg_cmd),
cwd=portage._unicode_encode(
self.repo.location),
**self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! hg clone error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
return (os.EX_OK, True)
def _get_target(self):
global VERSION
if VERSION is not self:
return VERSION
if os.path.isdir(os.path.join(PORTAGE_BASE_PATH, ".git")):
encoding = _encodings["fs"]
cmd = [
BASH_BINARY,
"-c",
(
"cd %s ; git describe --match 'repoman-*' || exit $? ; "
+ 'if [ -n "`git diff-index --name-only --diff-filter=M HEAD`" ] ; '
+ "then echo modified ; git rev-list --format=%%ct -n 1 HEAD ; fi ; "
+ "exit 0"
)
% _shell_quote(PORTAGE_BASE_PATH),
]
cmd = [
_unicode_encode(x, encoding=encoding, errors="strict") for x in cmd
]
proc = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
)
output = _unicode_decode(proc.communicate()[0], encoding=encoding)
status = proc.wait()
if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK:
output_lines = output.splitlines()
if output_lines:
version_split = output_lines[0].split("-")
if len(version_split) > 1:
VERSION = version_split[1]
patchlevel = False
if len(version_split) > 2:
patchlevel = True
VERSION = "%s_p%s" % (VERSION, version_split[2])
if len(output_lines) > 1 and output_lines[1] == "modified":
head_timestamp = None
if len(output_lines) > 3:
try:
head_timestamp = int(output_lines[3])
except ValueError:
pass
timestamp = int(time.time())
if (
head_timestamp is not None
and timestamp > head_timestamp
):
timestamp = timestamp - head_timestamp
if not patchlevel:
VERSION = "%s_p0" % (VERSION,)
VERSION = "%s_p%d" % (VERSION, timestamp)
return VERSION
else:
print("NO output lines :(")
VERSION = "HEAD"
return VERSION
def update(self):
''' Update existing git repository, and ignore the syncuri. We are
going to trust the user and assume that the user is in the branch
that he/she wants updated. We'll let the user manage branches with
git directly.
'''
git_cmd_opts = ""
quiet = self.settings.get("PORTAGE_QUIET") == "1"
if quiet:
git_cmd_opts += " --quiet"
if self.repo.module_specific_options.get('sync-git-pull-extra-opts'):
git_cmd_opts += " %s" % self.repo.module_specific_options['sync-git-pull-extra-opts']
if self.repo.sync_depth is None:
git_cmd = "%s pull%s" % (self.bin_command, git_cmd_opts)
else:
# Since the default merge strategy typically fails when
# the depth is not unlimited, use `git fetch` followed by
# `git reset --merge`.
remote_branch = portage._unicode_decode(
subprocess.check_output([self.bin_command, 'rev-parse',
'--abbrev-ref', '--symbolic-full-name', '@{upstream}'],
cwd=portage._unicode_encode(self.repo.location))).rstrip('\n')
git_cmd_opts += " --depth %d" % self.repo.sync_depth
git_cmd = "%s fetch %s%s" % (self.bin_command,
remote_branch.partition('/')[0], git_cmd_opts)
writemsg_level(git_cmd + "\n")
rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
previous_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(self.repo.location))
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), git_cmd),
**self.spawn_kwargs)
if exitcode == os.EX_OK and self.repo.sync_depth is not None:
reset_cmd = [self.bin_command, 'reset', '--merge', remote_branch]
if quiet:
reset_cmd.append('--quiet')
exitcode = subprocess.call(reset_cmd,
cwd=portage._unicode_encode(self.repo.location))
if exitcode != os.EX_OK:
msg = "!!! git pull error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
current_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(self.repo.location))
return (os.EX_OK, current_rev != previous_rev)
def update(self):
''' Update existing git repository, and ignore the syncuri. We are
going to trust the user and assume that the user is in the branch
that he/she wants updated. We'll let the user manage branches with
git directly.
'''
if not self.has_bin:
return (1, False)
git_cmd_opts = ""
if self.repo.module_specific_options.get('sync-git-env'):
shlexed_env = shlex_split(
self.repo.module_specific_options['sync-git-env'])
env = dict(
(k, v) for k, _, v in (assignment.partition('=')
for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(env)
if self.repo.module_specific_options.get('sync-git-pull-env'):
shlexed_env = shlex_split(
self.repo.module_specific_options['sync-git-pull-env'])
pull_env = dict(
(k, v) for k, _, v in (assignment.partition('=')
for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(pull_env)
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
if self.repo.module_specific_options.get('sync-git-pull-extra-opts'):
git_cmd_opts += " %s" % self.repo.module_specific_options[
'sync-git-pull-extra-opts']
git_cmd = "%s pull%s" % (self.bin_command, git_cmd_opts)
writemsg_level(git_cmd + "\n")
rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
previous_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(
self.repo.location))
exitcode = portage.process.spawn_bash(
"cd %s ; exec %s" %
(portage._shell_quote(self.repo.location), git_cmd),
**self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! git pull error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
if not self.verify_head():
return (1, False)
current_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(
self.repo.location))
return (os.EX_OK, current_rev != previous_rev)
def _start(self):
settings = self.settings
portage_bin_path = settings["PORTAGE_BIN_PATH"]
misc_sh_binary = os.path.join(portage_bin_path,
os.path.basename(portage.const.MISC_SH_BINARY))
self.args = [portage._shell_quote(misc_sh_binary)] + self.commands
if self.logfile is None:
self.logfile = settings.get("PORTAGE_LOG_FILE")
AbstractEbuildProcess._start(self)
def _start(self):
settings = self.settings
portage_bin_path = settings["PORTAGE_BIN_PATH"]
misc_sh_binary = os.path.join(
portage_bin_path, os.path.basename(portage.const.MISC_SH_BINARY))
self.args = [portage._shell_quote(misc_sh_binary)] + self.commands
if self.logfile is None:
self.logfile = settings.get("PORTAGE_LOG_FILE")
AbstractEbuildProcess._start(self)
def _start(self):
tar_options = ""
if "xattr" in self.features:
process = subprocess.Popen(["tar", "--help"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output = process.communicate()[0]
if b"--xattrs" in output:
tar_options = ["--xattrs", "--xattrs-include='*'"]
for x in portage.util.shlex_split(self.env.get("PORTAGE_XATTR_EXCLUDE", "")):
tar_options.append(portage._shell_quote("--xattrs-exclude=%s" % x))
tar_options = " ".join(tar_options)
decomp_cmd = _decompressors.get(
compression_probe(self.pkg_path))
if decomp_cmd is None:
self.scheduler.output("!!! %s\n" %
_("File compression header unrecognized: %s") %
self.pkg_path, log_path=self.logfile,
background=self.background, level=logging.ERROR)
self.returncode = 1
self._async_wait()
return
# Add -q to decomp_cmd opts, in order to avoid "trailing garbage
# after EOF ignored" warning messages due to xpak trailer.
# SIGPIPE handling (128 + SIGPIPE) should be compatible with
# assert_sigpipe_ok() that's used by the ebuild unpack() helper.
self.args = [self._shell_binary, "-c",
("%s -cq -- %s | tar -xp %s -C %s -f - ; " + \
"p=(${PIPESTATUS[@]}) ; " + \
"if [[ ${p[0]} != 0 && ${p[0]} != %d ]] ; then " % (128 + signal.SIGPIPE) + \
"echo bzip2 failed with status ${p[0]} ; exit ${p[0]} ; fi ; " + \
"if [ ${p[1]} != 0 ] ; then " + \
"echo tar failed with status ${p[1]} ; exit ${p[1]} ; fi ; " + \
"exit 0 ;") % \
(decomp_cmd,
portage._shell_quote(self.pkg_path),
tar_options,
portage._shell_quote(self.image_dir))]
SpawnProcess._start(self)
def new(self, **kwargs):
'''Do the initial clone of the repository'''
if kwargs:
self._kwargs(kwargs)
try:
if not os.path.exists(self.repo.location):
os.makedirs(self.repo.location)
self.logger(self.xterm_titles,
'Created new directory %s' % self.repo.location)
except IOError:
return (1, False)
sync_uri = self.repo.sync_uri
if sync_uri.startswith("file://"):
sync_uri = sync_uri[6:]
git_cmd_opts = ""
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
if self.repo.sync_depth is not None:
git_cmd_opts += " --depth %d" % self.repo.sync_depth
if self.repo.module_specific_options.get('sync-git-clone-extra-opts'):
git_cmd_opts += " %s" % self.repo.module_specific_options[
'sync-git-clone-extra-opts']
git_cmd = "%s clone%s %s ." % (self.bin_command, git_cmd_opts,
portage._shell_quote(sync_uri))
writemsg_level(git_cmd + "\n")
exitcode = portage.process.spawn_bash(
"cd %s ; exec %s" %
(portage._shell_quote(self.repo.location), git_cmd),
**self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! git clone error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
return (os.EX_OK, True)
def _start(self):
settings = self.settings
settings.pop("EBUILD_PHASE", None)
portage_bin_path = settings["PORTAGE_BIN_PATH"]
misc_sh_binary = os.path.join(portage_bin_path,
os.path.basename(portage.const.MISC_SH_BINARY))
self.args = [portage._shell_quote(misc_sh_binary)] + self.commands
self.logfile = settings.get("PORTAGE_LOG_FILE")
_doebuild_exit_status_unlink(
settings.get("EBUILD_EXIT_STATUS_FILE"))
AbstractEbuildProcess._start(self)
def thick_manifest(self, myupdates, myheaders, no_expansion, expansion):
if self.vcs_settings.vcs == 'cvs':
headerstring = "'\$(Header|Id).*\$'"
elif self.vcs_settings.vcs == "svn":
svn_keywords = dict((k.lower(), k) for k in [
"Rev",
"Revision",
"LastChangedRevision",
"Date",
"LastChangedDate",
"Author",
"LastChangedBy",
"URL",
"HeadURL",
"Id",
"Header",
])
for myfile in myupdates:
# for CVS, no_expansion contains files that are excluded from expansion
if self.vcs_settings.vcs == "cvs":
if myfile in no_expansion:
continue
# for SVN, expansion contains files that are included in expansion
elif self.vcs_settings.vcs == "svn":
if myfile not in expansion:
continue
# Subversion keywords are case-insensitive
# in svn:keywords properties,
# but case-sensitive in contents of files.
enabled_keywords = []
for k in expansion[myfile]:
keyword = svn_keywords.get(k.lower())
if keyword is not None:
enabled_keywords.append(keyword)
headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
myout = repoman_getstatusoutput(
"egrep -q %s %s" %
(headerstring, portage._shell_quote(myfile)))
if myout[0] == 0:
myheaders.append(myfile)
print("%s have headers that will change." % green(str(len(myheaders))))
print("* Files with headers will"
" cause the manifests to be changed and committed separately.")
def thick_manifest(self, myupdates, myheaders, no_expansion, expansion):
if self.vcs_settings.vcs == 'cvs':
headerstring = "'\$(Header|Id).*\$'"
elif self.vcs_settings.vcs == "svn":
svn_keywords = dict((k.lower(), k) for k in [
"Rev",
"Revision",
"LastChangedRevision",
"Date",
"LastChangedDate",
"Author",
"LastChangedBy",
"URL",
"HeadURL",
"Id",
"Header",
])
for myfile in myupdates:
# for CVS, no_expansion contains files that are excluded from expansion
if self.vcs_settings.vcs == "cvs":
if myfile in no_expansion:
continue
# for SVN, expansion contains files that are included in expansion
elif self.vcs_settings.vcs == "svn":
if myfile not in expansion:
continue
# Subversion keywords are case-insensitive
# in svn:keywords properties,
# but case-sensitive in contents of files.
enabled_keywords = []
for k in expansion[myfile]:
keyword = svn_keywords.get(k.lower())
if keyword is not None:
enabled_keywords.append(keyword)
headerstring = "'\$(%s).*\$'" % "|".join(enabled_keywords)
myout = repoman_getstatusoutput(
"egrep -q %s %s" % (headerstring, portage._shell_quote(myfile)))
if myout[0] == 0:
myheaders.append(myfile)
print("%s have headers that will change." % green(str(len(myheaders))))
print(
"* Files with headers will"
" cause the manifests to be changed and committed separately.")
def exists(self, **kwargs):
'''Tests whether the repo actually exists'''
if kwargs:
self._kwargs(kwargs)
elif not self.repo:
return False
if not os.path.exists(self.repo.location):
return False
exitcode = portage.process.spawn_bash("cd %s ; git rev-parse" %\
(portage._shell_quote(self.repo.location),),
**portage._native_kwargs(self.spawn_kwargs))
if exitcode == 128:
return False
return True
def _svn_upgrade(self):
"""
Internal function which performs an svn upgrade on the repo
@return: tuple of return code (0=success), whether the cache
needs to be updated
@rtype: (int, bool)
"""
exitcode = portage.process.spawn_bash(
"cd %s; exec svn upgrade" %
(portage._shell_quote(self.repo.location), ), **self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! svn upgrade error; exiting."
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR)
return exitcode
def _svn_upgrade(self): """ Internal function which performs an svn upgrade on the repo @return: tuple of return code (0=success), whether the cache needs to be updated @rtype: (int, bool) """ exitcode = portage.process.spawn_bash( "cd %s; exec svn upgrade" % (portage._shell_quote(self.repo.location),), **self.spawn_kwargs) if exitcode != os.EX_OK: msg = "!!! svn upgrade error; exiting." self.logger(self.xterm_titles, msg) writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR) return exitcode
def update(self):
''' Update existing git repository, and ignore the syncuri. We are
going to trust the user and assume that the user is in the branch
that he/she wants updated. We'll let the user manage branches with
git directly.
'''
if not self.has_bin:
return (1, False)
git_cmd_opts = ""
if self.repo.module_specific_options.get('sync-git-env'):
shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-env'])
env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(env)
if self.repo.module_specific_options.get('sync-git-pull-env'):
shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-pull-env'])
pull_env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(pull_env)
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
if self.repo.module_specific_options.get('sync-git-pull-extra-opts'):
git_cmd_opts += " %s" % self.repo.module_specific_options['sync-git-pull-extra-opts']
git_cmd = "%s pull%s" % (self.bin_command, git_cmd_opts)
writemsg_level(git_cmd + "\n")
rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
previous_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(self.repo.location))
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), git_cmd),
**self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! git pull error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
if not self.verify_head():
return (1, False)
current_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(self.repo.location))
return (os.EX_OK, current_rev != previous_rev)
def _get_target(self):
global VERSION
if VERSION is not self:
return VERSION
if os.path.isdir(os.path.join(PORTAGE_BASE_PATH, '.git')):
encoding = _encodings['fs']
cmd = [BASH_BINARY, "-c", ("cd %s ; git describe --match 'repoman-*' || exit $? ; " + \
"if [ -n \"`git diff-index --name-only --diff-filter=M HEAD`\" ] ; " + \
"then echo modified ; git rev-list --format=%%ct -n 1 HEAD ; fi ; " + \
"exit 0") % _shell_quote(PORTAGE_BASE_PATH)]
cmd = [_unicode_encode(x, encoding=encoding, errors='strict')
for x in cmd]
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = _unicode_decode(proc.communicate()[0], encoding=encoding)
status = proc.wait()
if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK:
output_lines = output.splitlines()
if output_lines:
version_split = output_lines[0].split('-')
if len(version_split) > 1:
VERSION = version_split[1]
patchlevel = False
if len(version_split) > 2:
patchlevel = True
VERSION = "%s_p%s" % (VERSION, version_split[2])
if len(output_lines) > 1 and output_lines[1] == 'modified':
head_timestamp = None
if len(output_lines) > 3:
try:
head_timestamp = long(output_lines[3])
except ValueError:
pass
timestamp = long(time.time())
if head_timestamp is not None and timestamp > head_timestamp:
timestamp = timestamp - head_timestamp
if not patchlevel:
VERSION = "%s_p0" % (VERSION,)
VERSION = "%s_p%d" % (VERSION, timestamp)
return VERSION
else:
print("NO output lines :(")
VERSION = 'HEAD'
return VERSION
def update(self):
"""
Internal function to update an existing CVS repository
@return: tuple of return code (0=success), whether the cache
needs to be updated
@rtype: (int, bool)
"""
#cvs update
exitcode = portage.process.spawn_bash(
"cd %s; exec cvs -z0 -q update -dP" % \
(portage._shell_quote(self.repo.location),),
**portage._native_kwargs(self.spawn_kwargs))
if exitcode != os.EX_OK:
msg = "!!! cvs update error; exiting."
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR)
return (exitcode, False)
def _start(self):
try:
with io.open(
_unicode_encode(
os.path.join(
self.settings["PORTAGE_BUILDDIR"],
"build-info",
"PKG_INSTALL_MASK",
),
encoding=_encodings["fs"],
errors="strict",
),
mode="r",
encoding=_encodings["repo.content"],
errors="replace",
) as f:
self._pkg_install_mask = InstallMask(f.read())
except EnvironmentError:
self._pkg_install_mask = None
if self._pkg_install_mask:
self._proot = os.path.join(self.settings["T"], "packaging")
self._start_task(
SpawnProcess(
args=[
self._shell_binary,
"-e",
"-c",
(
"rm -rf {PROOT}; "
'cp -pPR $(cp --help | grep -q -- "^[[:space:]]*-l," && echo -l)'
' "${{D}}" {PROOT}'
).format(PROOT=portage._shell_quote(self._proot)),
],
background=self.background,
env=self.settings.environ(),
scheduler=self.scheduler,
logfile=self.logfile,
),
self._copy_proot_exit,
)
else:
self._proot = self.settings["D"]
self._start_package_phase()
def update(self):
''' Update existing git repository, and ignore the syncuri. We are
going to trust the user and assume that the user is in the branch
that he/she wants updated. We'll let the user manage branches with
git directly.
'''
git_cmd = "%s pull" % self.bin_command
writemsg_level(git_cmd + "\n")
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), git_cmd),
**portage._native_kwargs(self.spawn_kwargs))
if exitcode != os.EX_OK:
msg = "!!! git pull error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
return (os.EX_OK, True)
def update(self): """ Internal function to update an existing CVS repository @return: tuple of return code (0=success), whether the cache needs to be updated @rtype: (int, bool) """ #cvs update exitcode = portage.process.spawn_bash( "cd %s; exec cvs -z0 -q update -dP" % \ (portage._shell_quote(self.repo.location),), **portage._native_kwargs(self.spawn_kwargs)) if exitcode != os.EX_OK: msg = "!!! cvs update error; exiting." self.logger(self.xterm_titles, msg) writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR) return (exitcode, False)
def _start(self):
try:
with io.open(_unicode_encode(
os.path.join(self.settings["PORTAGE_BUILDDIR"],
"build-info", "PKG_INSTALL_MASK"),
encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['repo.content'],
errors='replace') as f:
self._pkg_install_mask = InstallMask(f.read())
except EnvironmentError:
self._pkg_install_mask = None
if self._pkg_install_mask:
self._proot = os.path.join(self.settings['T'], 'packaging')
self._start_task(SpawnProcess(
args=[self._shell_binary, '-e', '-c', ('rm -rf {PROOT}; '
'cp -pPR $(cp --help | grep -q -- "^[[:space:]]*-l," && echo -l)'
' "${{D}}" {PROOT}').format(PROOT=portage._shell_quote(self._proot))],
background=self.background, env=self.settings.environ(),
scheduler=self.scheduler, logfile=self.logfile),
self._copy_proot_exit)
else:
self._proot = self.settings['D']
self._start_package_phase()
def _userpriv_test_write_file(settings, file_path):
"""
Drop privileges and try to open a file for writing. The file may or
may not exist, and the parent directory is assumed to exist. The file
is removed before returning.
@param settings: A config instance which is passed to _spawn_fetch()
@param file_path: A file path to open and write.
@return: True if write succeeds, False otherwise.
"""
global _userpriv_test_write_file_cache, _userpriv_test_write_cmd_script
rval = _userpriv_test_write_file_cache.get(file_path)
if rval is not None:
return rval
args = [BASH_BINARY, "-c", _userpriv_test_write_cmd_script % \
{"file_path" : _shell_quote(file_path)}]
returncode = _spawn_fetch(settings, args)
rval = returncode == os.EX_OK
_userpriv_test_write_file_cache[file_path] = rval
return rval
def _userpriv_test_write_file(settings, file_path):
"""
Drop privileges and try to open a file for writing. The file may or
may not exist, and the parent directory is assumed to exist. The file
is removed before returning.
@param settings: A config instance which is passed to _spawn_fetch()
@param file_path: A file path to open and write.
@return: True if write succeeds, False otherwise.
"""
global _userpriv_test_write_file_cache, _userpriv_test_write_cmd_script
rval = _userpriv_test_write_file_cache.get(file_path)
if rval is not None:
return rval
args = [BASH_BINARY, "-c", _userpriv_test_write_cmd_script % \
{"file_path" : _shell_quote(file_path)}]
returncode = _spawn_fetch(settings, args)
rval = returncode == os.EX_OK
_userpriv_test_write_file_cache[file_path] = rval
return rval
def testSyncLocal(self):
debug = False
skip_reason = self._must_skip()
if skip_reason:
self.portage_skip = skip_reason
self.assertFalse(True, skip_reason)
return
repos_conf = textwrap.dedent("""
[DEFAULT]
%(default_keys)s
[test_repo]
location = %(EPREFIX)s/var/repositories/test_repo
sync-type = %(sync-type)s
sync-depth = %(sync-depth)s
sync-uri = file://%(EPREFIX)s/var/repositories/test_repo_sync
sync-rcu = %(sync-rcu)s
sync-rcu-store-dir = %(EPREFIX)s/var/repositories/test_repo_rcu_storedir
auto-sync = %(auto-sync)s
%(repo_extra_keys)s
""")
profile = {
"eapi": ("5", ),
"package.use.stable.mask": ("dev-libs/A flag", )
}
ebuilds = {"dev-libs/A-0": {}}
user_config = {'make.conf': ('FEATURES="metadata-transfer"', )}
playground = ResolverPlayground(ebuilds=ebuilds,
profile=profile,
user_config=user_config,
debug=debug)
settings = playground.settings
eprefix = settings["EPREFIX"]
eroot = settings["EROOT"]
homedir = os.path.join(eroot, "home")
distdir = os.path.join(eprefix, "distdir")
repo = settings.repositories["test_repo"]
metadata_dir = os.path.join(repo.location, "metadata")
cmds = {}
for cmd in ("emerge", "emaint"):
for bindir in (self.bindir, self.sbindir):
path = os.path.join(bindir, cmd)
if os.path.exists(path):
cmds[cmd] = (portage._python_interpreter, "-b", "-Wd",
path)
break
else:
raise AssertionError('%s binary not found in %s or %s' %
(cmd, self.bindir, self.sbindir))
git_binary = find_binary("git")
git_cmd = (git_binary, )
committer_name = "Gentoo Dev"
committer_email = "*****@*****.**"
def repos_set_conf(sync_type,
dflt_keys=None,
xtra_keys=None,
auto_sync="yes",
sync_rcu=False,
sync_depth=None):
env["PORTAGE_REPOSITORIES"] = repos_conf % {\
"EPREFIX": eprefix, "sync-type": sync_type,
"sync-depth": 0 if sync_depth is None else sync_depth,
"sync-rcu": "yes" if sync_rcu else "no",
"auto-sync": auto_sync,
"default_keys": "" if dflt_keys is None else dflt_keys,
"repo_extra_keys": "" if xtra_keys is None else xtra_keys}
def alter_ebuild():
with open(
os.path.join(repo.location + "_sync", "dev-libs", "A",
"A-0.ebuild"), "a") as f:
f.write("\n")
bump_timestamp()
def bump_timestamp():
bump_timestamp.timestamp += datetime.timedelta(seconds=1)
with open(
os.path.join(repo.location + '_sync', 'metadata',
'timestamp.chk'), 'w') as f:
f.write(
bump_timestamp.timestamp.strftime(
'%s\n' % TIMESTAMP_FORMAT, ))
bump_timestamp.timestamp = datetime.datetime.utcnow()
bump_timestamp_cmds = ((homedir, bump_timestamp), )
sync_cmds = (
(homedir, cmds["emerge"] + ("--sync", )),
(homedir, lambda: self.assertTrue(
os.path.exists(os.path.join(repo.location, "dev-libs", "A")),
"dev-libs/A expected, but missing")),
(homedir, cmds["emaint"] + ("sync", "-A")),
)
sync_cmds_auto_sync = (
(homedir, lambda: repos_set_conf("rsync", auto_sync="no")),
(homedir, cmds["emerge"] + ("--sync", )),
(homedir, lambda: self.assertFalse(
os.path.exists(os.path.join(repo.location, "dev-libs", "A")),
"dev-libs/A found, expected missing")),
(homedir, lambda: repos_set_conf("rsync", auto_sync="yes")),
)
rename_repo = (
(homedir,
lambda: os.rename(repo.location, repo.location + "_sync")), )
rsync_opts_repos = (
(homedir, alter_ebuild),
(homedir, lambda: repos_set_conf(
"rsync", None,
"sync-rsync-extra-opts = --backup --backup-dir=%s" %
_shell_quote(repo.location + "_back"))),
(homedir, cmds['emerge'] + ("--sync", )),
(homedir,
lambda: self.assertTrue(os.path.exists(repo.location + "_back"))),
(homedir, lambda: shutil.rmtree(repo.location + "_back")),
(homedir, lambda: repos_set_conf("rsync")),
)
rsync_opts_repos_default = (
(homedir, alter_ebuild),
(homedir, lambda: repos_set_conf(
"rsync", "sync-rsync-extra-opts = --backup --backup-dir=%s" %
_shell_quote(repo.location + "_back"))),
(homedir, cmds['emerge'] + ("--sync", )),
(homedir,
lambda: self.assertTrue(os.path.exists(repo.location + "_back"))),
(homedir, lambda: shutil.rmtree(repo.location + "_back")),
(homedir, lambda: repos_set_conf("rsync")),
)
rsync_opts_repos_default_ovr = (
(homedir, alter_ebuild),
(homedir, lambda: repos_set_conf(
"rsync", "sync-rsync-extra-opts = --backup --backup-dir=%s" %
_shell_quote(repo.location + "_back_nowhere"),
"sync-rsync-extra-opts = --backup --backup-dir=%s" %
_shell_quote(repo.location + "_back"))),
(homedir, cmds['emerge'] + ("--sync", )),
(homedir,
lambda: self.assertTrue(os.path.exists(repo.location + "_back"))),
(homedir, lambda: shutil.rmtree(repo.location + "_back")),
(homedir, lambda: repos_set_conf("rsync")),
)
rsync_opts_repos_default_cancel = (
(homedir, alter_ebuild),
(homedir, lambda: repos_set_conf(
"rsync", "sync-rsync-extra-opts = --backup --backup-dir=%s" %
_shell_quote(repo.location + "_back_nowhere"
), "sync-rsync-extra-opts = ")),
(homedir, cmds['emerge'] + ("--sync", )),
(homedir,
lambda: self.assertFalse(os.path.exists(repo.location + "_back"))
),
(homedir, lambda: repos_set_conf("rsync")),
)
delete_repo_location = (
(homedir, lambda: shutil.rmtree(repo.user_location)),
(homedir, lambda: os.mkdir(repo.user_location)),
)
revert_rcu_layout = (
(homedir,
lambda: os.rename(repo.user_location, repo.user_location + '.bak')
),
(homedir,
lambda: os.rename(os.path.realpath(repo.user_location + '.bak'),
repo.user_location)),
(homedir, lambda: os.unlink(repo.user_location + '.bak')),
(homedir,
lambda: shutil.rmtree(repo.user_location + '_rcu_storedir')),
)
upstream_git_commit = (
(
repo.location + "_sync",
git_cmd +
('commit', '--allow-empty', '-m', 'test empty commit'),
),
(
repo.location + "_sync",
git_cmd +
('commit', '--allow-empty', '-m', 'test empty commit 2'),
),
)
delete_sync_repo = ((homedir,
lambda: shutil.rmtree(repo.location + "_sync")), )
git_repo_create = (
(repo.location, git_cmd + (
"config",
"--global",
"user.name",
committer_name,
)),
(repo.location, git_cmd + (
"config",
"--global",
"user.email",
committer_email,
)),
(repo.location, git_cmd + ("init-db", )),
(repo.location, git_cmd + ("add", ".")),
(repo.location,
git_cmd + ("commit", "-a", "-m", "add whole repo")),
)
sync_type_git = ((homedir, lambda: repos_set_conf("git")), )
sync_type_git_shallow = ((
homedir, lambda: repos_set_conf("git", sync_depth=1)), )
sync_rsync_rcu = ((homedir,
lambda: repos_set_conf("rsync", sync_rcu=True)), )
pythonpath = os.environ.get("PYTHONPATH")
if pythonpath is not None and not pythonpath.strip():
pythonpath = None
if pythonpath is not None and \
pythonpath.split(":")[0] == PORTAGE_PYM_PATH:
pass
else:
if pythonpath is None:
pythonpath = ""
else:
pythonpath = ":" + pythonpath
pythonpath = PORTAGE_PYM_PATH + pythonpath
env = {
"PORTAGE_OVERRIDE_EPREFIX":
eprefix,
"DISTDIR":
distdir,
"GENTOO_COMMITTER_NAME":
committer_name,
"GENTOO_COMMITTER_EMAIL":
committer_email,
"HOME":
homedir,
"PATH":
os.environ["PATH"],
"PORTAGE_GRPNAME":
os.environ["PORTAGE_GRPNAME"],
"PORTAGE_USERNAME":
os.environ["PORTAGE_USERNAME"],
"PYTHONDONTWRITEBYTECODE":
os.environ.get("PYTHONDONTWRITEBYTECODE", ""),
"PYTHONPATH":
pythonpath,
}
repos_set_conf("rsync")
if os.environ.get("SANDBOX_ON") == "1":
# avoid problems from nested sandbox instances
env["FEATURES"] = "-sandbox -usersandbox"
dirs = [homedir, metadata_dir]
try:
for d in dirs:
ensure_dirs(d)
timestamp_path = os.path.join(metadata_dir, 'timestamp.chk')
with open(timestamp_path, 'w') as f:
f.write(
bump_timestamp.timestamp.strftime(
'%s\n' % TIMESTAMP_FORMAT, ))
if debug:
# The subprocess inherits both stdout and stderr, for
# debugging purposes.
stdout = None
else:
# The subprocess inherits stderr so that any warnings
# triggered by python -Wd will be visible.
stdout = subprocess.PIPE
for cwd, cmd in rename_repo + sync_cmds_auto_sync + sync_cmds + \
rsync_opts_repos + rsync_opts_repos_default + \
rsync_opts_repos_default_ovr + rsync_opts_repos_default_cancel + \
bump_timestamp_cmds + sync_rsync_rcu + sync_cmds + revert_rcu_layout + \
delete_repo_location + sync_cmds + sync_cmds + \
bump_timestamp_cmds + sync_cmds + revert_rcu_layout + \
delete_sync_repo + git_repo_create + sync_type_git + \
rename_repo + sync_cmds + upstream_git_commit + sync_cmds + \
sync_type_git_shallow + upstream_git_commit + sync_cmds:
if hasattr(cmd, '__call__'):
cmd()
continue
abs_cwd = os.path.join(repo.location, cwd)
proc = subprocess.Popen(cmd,
cwd=abs_cwd,
env=env,
stdout=stdout)
if debug:
proc.wait()
else:
output = proc.stdout.readlines()
proc.wait()
proc.stdout.close()
if proc.returncode != os.EX_OK:
for line in output:
sys.stderr.write(_unicode_decode(line))
self.assertEqual(os.EX_OK, proc.returncode,
"%s failed in %s" % (
cmd,
cwd,
))
finally:
playground.cleanup()
def package_from_ebuild(ebuild):
pf = None
if ebuild.endswith(".ebuild"):
pf = os.path.basename(ebuild)[:-7]
else:
return False
if not os.path.isabs(ebuild):
mycwd = os.getcwd()
# Try to get the non-canonical path from the PWD evironment variable,
# since the canonical path returned from os.getcwd() may may be
# unusable in cases where the directory stucture is built from
# symlinks.
pwd = os.environ.get('PWD', '')
if sys.hexversion < 0x3000000:
pwd = _unicode_decode(pwd,
encoding=_encodings['content'],
errors='strict')
if pwd and pwd != mycwd and \
os.path.realpath(pwd) == mycwd:
mycwd = portage.normalize_path(pwd)
ebuild = os.path.join(mycwd, ebuild)
ebuild = portage.normalize_path(ebuild)
# portdbapi uses the canonical path for the base of the portage tree, but
# subdirectories of the base can be built from symlinks (like crossdev
# does).
ebuild_portdir = os.path.realpath(
os.path.dirname(os.path.dirname(os.path.dirname(ebuild))))
ebuild = os.path.join(ebuild_portdir, *ebuild.split(os.path.sep)[-3:])
vdb_path = os.path.join(portage.settings['ROOT'], VDB_PATH)
# Make sure that portdb.findname() returns the correct ebuild.
if ebuild_portdir != vdb_path and \
ebuild_portdir not in portage.portdb.porttrees:
if sys.hexversion >= 0x3000000:
os.environ["PORTDIR_OVERLAY"] = \
os.environ.get("PORTDIR_OVERLAY", "") + \
" " + _shell_quote(ebuild_portdir)
else:
os.environ["PORTDIR_OVERLAY"] = \
os.environ.get("PORTDIR_OVERLAY", "") + \
" " + _unicode_encode(_shell_quote(ebuild_portdir),
encoding=_encodings['content'], errors='strict')
portage.close_portdbapi_caches()
imp.reload(portage)
del portage.portdb.porttrees[1:]
if ebuild_portdir != portage.portdb.porttree_root:
portage.portdb.porttrees.append(ebuild_portdir)
if not os.path.exists(ebuild):
return False
ebuild_split = ebuild.split("/")
cpv = "%s/%s" % (ebuild_split[-3], pf)
if not portage.catpkgsplit(cpv):
return False
if ebuild.startswith(os.path.join(portage.root, portage.const.VDB_PATH)):
mytree = "vartree"
portage_ebuild = portage.db[portage.root][mytree].dbapi.findname(cpv)
if os.path.realpath(portage_ebuild) != ebuild:
return False
else:
mytree = "porttree"
portage_ebuild = portage.portdb.findname(cpv)
if not portage_ebuild or portage_ebuild != ebuild:
return False
return cpv
def testPortdbCache(self):
debug = False
ebuilds = {
"dev-libs/A-1": {},
"dev-libs/A-2": {},
"sys-apps/B-1": {},
"sys-apps/B-2": {},
}
playground = ResolverPlayground(ebuilds=ebuilds, debug=debug)
settings = playground.settings
eprefix = settings["EPREFIX"]
test_repo_location = settings.repositories["test_repo"].location
user_config_dir = os.path.join(eprefix, USER_CONFIG_PATH)
metadata_dir = os.path.join(test_repo_location, "metadata")
md5_cache_dir = os.path.join(metadata_dir, "md5-cache")
pms_cache_dir = os.path.join(metadata_dir, "cache")
layout_conf_path = os.path.join(metadata_dir, "layout.conf")
portage_python = portage._python_interpreter
egencache_cmd = (
portage_python,
"-b",
"-Wd",
os.path.join(self.bindir, "egencache"),
"--update-manifests",
"--sign-manifests=n",
"--repo",
"test_repo",
"--repositories-configuration",
settings.repositories.config_string(),
)
python_cmd = (portage_python, "-b", "-Wd", "-c")
test_commands = (
(lambda: not os.path.exists(pms_cache_dir),),
(lambda: not os.path.exists(md5_cache_dir),),
python_cmd
+ (
textwrap.dedent(
"""
import os, sys, portage
if portage.portdb.repositories['test_repo'].location in portage.portdb._pregen_auxdb:
sys.exit(1)
"""
),
),
egencache_cmd + ("--update",),
(lambda: not os.path.exists(pms_cache_dir),),
(lambda: os.path.exists(md5_cache_dir),),
python_cmd
+ (
textwrap.dedent(
"""
import os, sys, portage
if portage.portdb.repositories['test_repo'].location not in portage.portdb._pregen_auxdb:
sys.exit(1)
"""
),
),
python_cmd
+ (
textwrap.dedent(
"""
import os, sys, portage
from portage.cache.flat_hash import md5_database
if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.repositories['test_repo'].location], md5_database):
sys.exit(1)
"""
),
),
(
BASH_BINARY,
"-c",
"echo %s > %s"
% tuple(
map(
portage._shell_quote,
(
"cache-formats = md5-dict pms",
layout_conf_path,
),
)
),
),
egencache_cmd + ("--update",),
(lambda: os.path.exists(md5_cache_dir),),
python_cmd
+ (
textwrap.dedent(
"""
import os, sys, portage
if portage.portdb.repositories['test_repo'].location not in portage.portdb._pregen_auxdb:
sys.exit(1)
"""
),
),
python_cmd
+ (
textwrap.dedent(
"""
import os, sys, portage
from portage.cache.flat_hash import md5_database
if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.repositories['test_repo'].location], md5_database):
sys.exit(1)
"""
),
),
# Disable DeprecationWarnings, since the pms format triggers them
# in portdbapi._create_pregen_cache().
(
BASH_BINARY,
"-c",
"echo %s > %s"
% tuple(
map(
portage._shell_quote,
(
"cache-formats = pms md5-dict",
layout_conf_path,
),
)
),
),
(portage_python, "-b", "-Wd", "-Wi::DeprecationWarning", "-c")
+ (
textwrap.dedent(
"""
import os, sys, portage
if portage.portdb.repositories['test_repo'].location not in portage.portdb._pregen_auxdb:
sys.exit(1)
"""
),
),
(portage_python, "-b", "-Wd", "-Wi::DeprecationWarning", "-c")
+ (
textwrap.dedent(
"""
import os, sys, portage
from portage.cache.metadata import database as pms_database
if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.repositories['test_repo'].location], pms_database):
sys.exit(1)
"""
),
),
# Test auto-detection and preference for md5-cache when both
# cache formats are available but layout.conf is absent.
(BASH_BINARY, "-c", "rm %s" % portage._shell_quote(layout_conf_path)),
python_cmd
+ (
textwrap.dedent(
"""
import os, sys, portage
if portage.portdb.repositories['test_repo'].location not in portage.portdb._pregen_auxdb:
sys.exit(1)
"""
),
),
python_cmd
+ (
textwrap.dedent(
"""
import os, sys, portage
from portage.cache.flat_hash import md5_database
if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.repositories['test_repo'].location], md5_database):
sys.exit(1)
"""
),
),
)
pythonpath = os.environ.get("PYTHONPATH")
if pythonpath is not None and not pythonpath.strip():
pythonpath = None
if pythonpath is not None and pythonpath.split(":")[0] == PORTAGE_PYM_PATH:
pass
else:
if pythonpath is None:
pythonpath = ""
else:
pythonpath = ":" + pythonpath
pythonpath = PORTAGE_PYM_PATH + pythonpath
env = {
"PATH": os.environ.get("PATH", ""),
"PORTAGE_OVERRIDE_EPREFIX": eprefix,
"PORTAGE_PYTHON": portage_python,
"PORTAGE_REPOSITORIES": settings.repositories.config_string(),
"PYTHONDONTWRITEBYTECODE": os.environ.get("PYTHONDONTWRITEBYTECODE", ""),
"PYTHONPATH": pythonpath,
}
if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
env["__PORTAGE_TEST_HARDLINK_LOCKS"] = os.environ[
"__PORTAGE_TEST_HARDLINK_LOCKS"
]
dirs = [user_config_dir]
try:
for d in dirs:
ensure_dirs(d)
if debug:
# The subprocess inherits both stdout and stderr, for
# debugging purposes.
stdout = None
else:
# The subprocess inherits stderr so that any warnings
# triggered by python -Wd will be visible.
stdout = subprocess.PIPE
for i, args in enumerate(test_commands):
if hasattr(args[0], "__call__"):
self.assertTrue(args[0](), "callable at index %s failed" % (i,))
continue
proc = subprocess.Popen(args, env=env, stdout=stdout)
if debug:
proc.wait()
else:
output = proc.stdout.readlines()
proc.wait()
proc.stdout.close()
if proc.returncode != os.EX_OK:
for line in output:
sys.stderr.write(_unicode_decode(line))
self.assertEqual(
os.EX_OK,
proc.returncode,
"command %d failed with args %s"
% (
i,
args,
),
)
finally:
playground.cleanup()
def testShellQuote(self):
test_data = [
# String contains no special characters, should be preserved.
("abc","abc"),
# String contains whitespace, should be double-quoted to prevent word splitting.
("abc xyz","\"abc xyz\""),
("abc xyz","\"abc xyz\""),
(" abcxyz ","\" abcxyz \""),
("abc\txyz","\"abc\txyz\""),
("abc\t\txyz","\"abc\t\txyz\""),
("\tabcxyz\t","\"\tabcxyz\t\""),
("abc\nxyz","\"abc\nxyz\""),
("abc\n\nxyz","\"abc\n\nxyz\""),
("\nabcxyz\n","\"\nabcxyz\n\""),
# String contains > or <, should be double-quoted to prevent redirection.
("abc>xyz","\"abc>xyz\""),
("abc>>xyz","\"abc>>xyz\""),
(">abcxyz>","\">abcxyz>\""),
("abc<xyz","\"abc<xyz\""),
("abc<<xyz","\"abc<<xyz\""),
("<abcxyz<","\"<abcxyz<\""),
# String contains =, should be double-quoted to prevent assignment.
("abc=xyz","\"abc=xyz\""),
("abc==xyz","\"abc==xyz\""),
("=abcxyz=","\"=abcxyz=\""),
# String contains *, should be double-quoted to prevent globbing.
("abc*xyz","\"abc*xyz\""),
("abc**xyz","\"abc**xyz\""),
("*abcxyz*","\"*abcxyz*\""),
# String contains $, should be escaped to prevent parameter expansion.
# Also double-quoted, though not strictly needed.
("abc$xyz","\"abc\\$xyz\""),
("abc$$xyz","\"abc\\$\\$xyz\""),
("$abcxyz$","\"\\$abcxyz\\$\""),
# String contains `, should be escaped to prevent command substitution.
# Also double-quoted, though not strictly needed.
("abc`xyz","\"abc\\`xyz\""),
("abc``xyz","\"abc\\`\\`xyz\""),
("`abc`","\"\\`abc\\`\""),
# String contains \, should be escaped to prevent it from escaping
# the next character. Also double-quoted, though not strictly needed.
("abc\\xyz", "\"abc\\\\xyz\""),
("abc\\\\xyz", "\"abc\\\\\\\\xyz\""),
("\\abcxyz\\", "\"\\\\abcxyz\\\\\""),
# String contains ", should be escaped to prevent it from unexpectedly
# ending a previous double-quote or starting a new double-quote. Also
# double-quoted, though not strictly needed.
("abc\"xyz","\"abc\\\"xyz\""),
("abc\"\"xyz","\"abc\\\"\\\"xyz\""),
("\"abcxyz\"","\"\\\"abcxyz\\\"\""),
# String contains ', should be double-quoted to prevent it from unexpectedly
# ending a previous single-quote or starting a new single-quote.
("abc'xyz","\"abc'xyz\""),
("abc''xyz","\"abc''xyz\""),
("'abcxyz'","\"'abcxyz'\""),
# String contains ;, should be double-quoted to prevent command separation.
("abc;xyz","\"abc;xyz\""),
("abc;;xyz","\"abc;;xyz\""),
(";abcxyz;","\";abcxyz;\""),
# String contains &, should be double-quoted to prevent job control.
("abc&xyz","\"abc&xyz\""),
("abc&&xyz","\"abc&&xyz\""),
("&abcxyz&","\"&abcxyz&\""),
# String contains |, should be double-quoted to prevent piping.
("abc|xyz","\"abc|xyz\""),
("abc||xyz","\"abc||xyz\""),
("|abcxyz|","\"|abcxyz|\""),
# String contains (), should be double-quoted to prevent
# command group / array initialization.
("abc()xyz","\"abc()xyz\""),
("abc(())xyz","\"abc(())xyz\""),
("((abcxyz))","\"((abcxyz))\""),
# String contains {}. Parameter expansion of the form ${} is already
# rendered safe by escaping the $, but {} could also occur on its own,
# for example in a brace expansion such as filename.{ext1,ext2},
# so the string should be double-quoted.
("abc{}xyz","\"abc{}xyz\""),
("abc{{}}xyz","\"abc{{}}xyz\""),
("{{abcxyz}}","\"{{abcxyz}}\""),
# String contains [], should be double-quoted to prevent testing
("abc[]xyz","\"abc[]xyz\""),
("abc[[]]xyz","\"abc[[]]xyz\""),
("[[abcxyz]]","\"[[abcxyz]]\""),
# String contains #, should be double-quoted to prevent comment.
("#abc","\"#abc\""),
# String contains !, should be double-quoted to prevent e.g. history substitution.
("!abc","\"!abc\""),
# String contains ~, should be double-quoted to prevent home directory expansion.
("~abc","\"~abc\""),
# String contains ?, should be double-quoted to prevent globbing.
("abc?xyz","\"abc?xyz\""),
("abc??xyz","\"abc??xyz\""),
("?abcxyz?","\"?abcxyz?\""),
]
for (data,expected_result) in test_data:
result = _shell_quote(data)
self.assertEqual(result, expected_result)
