def _clean():
"""Cleans the root trial directory."""
if not TrialDir.SHOULD_LEAK:
logging.debug('Removing %s' % TrialDir.TRIAL_ROOT)
gclient_utils.RemoveDirectory(TrialDir.TRIAL_ROOT)
else:
logging.error('Leaking %s' % TrialDir.TRIAL_ROOT)
def DiffItem(filename, full_move=False, revision=None):
"""Diffs a single file.
Should be simple, eh? No it isn't.
Be sure to be in the appropriate directory before calling to have the
expected relative path.
full_move means that move or copy operations should completely recreate the
files, usually in the prospect to apply the patch for a try job."""
# If the user specified a custom diff command in their svn config file,
# then it'll be used when we do svn diff, which we don't want to happen
# since we want the unified diff. Using --diff-cmd=diff doesn't always
# work, since they can have another diff executable in their path that
# gives different line endings. So we use a bogus temp directory as the
# config directory, which gets around these problems.
bogus_dir = tempfile.mkdtemp()
try:
# Use "svn info" output instead of os.path.isdir because the latter fails
# when the file is deleted.
return SVN._DiffItemInternal(filename,
SVN.CaptureInfo(filename),
bogus_dir,
full_move=full_move,
revision=revision)
finally:
gclient_utils.RemoveDirectory(bogus_dir)
def tear_down(self):
"""Cleans the trial subdirectory for this instance."""
if not self.leak:
logging.debug('Removing %s' % self.root_dir)
gclient_utils.RemoveDirectory(self.root_dir)
else:
logging.error('Leaking %s' % self.root_dir)
self.root_dir = None
def set_up(self):
"""All late initialization comes here."""
# You can override self.TRIAL_ROOT.
if not self.TRIAL_ROOT:
# Was not yet initialized.
TrialDir.TRIAL_ROOT = os.path.realpath(tempfile.mkdtemp(prefix='trial'))
atexit.register(self._clean)
self.root_dir = os.path.join(TrialDir.TRIAL_ROOT, self.subdir)
gclient_utils.RemoveDirectory(self.root_dir)
os.makedirs(self.root_dir)
def Revert(cwd, callback=None, ignore_externals=False, no_ignore=False):
"""Reverts all svn modifications in cwd, including properties.
Deletes any modified files or directory.
A "svn update --revision BASE" call is required after to revive deleted
files.
"""
for file_status in SVN.CaptureStatus(None, cwd, no_ignore=no_ignore):
file_path = os.path.join(cwd, file_status[1])
if (ignore_externals and file_status[0][0] == 'X'
and file_status[0][1:].isspace()):
# Ignore externals.
logging.info('Ignoring external %s' % file_status[1])
continue
# This is the case where '! L .' is returned by 'svn status'. Just
# strip off the '/.'.
if file_path.endswith(os.path.sep + '.'):
file_path = file_path[:-2]
if callback:
callback(file_status)
if os.path.exists(file_path):
# svn revert is really stupid. It fails on inconsistent line-endings,
# on switched directories, etc. So take no chance and delete everything!
# In theory, it wouldn't be necessary for property-only change but then
# it'd have to look for switched directories, etc so it's not worth
# optimizing this use case.
if os.path.isfile(file_path) or os.path.islink(file_path):
logging.info('os.remove(%s)' % file_path)
os.remove(file_path)
elif os.path.isdir(file_path):
logging.info('RemoveDirectory(%s)' % file_path)
gclient_utils.RemoveDirectory(file_path)
else:
logging.critical(
('No idea what is %s.\nYou just found a bug in gclient'
', please ping [email protected] ASAP!') %
file_path)
if (file_status[0][0] in ('D', 'A', '!')
or not file_status[0][1:].isspace()):
# Added, deleted file requires manual intervention and require calling
# revert, like for properties.
if not os.path.isdir(cwd):
# '.' was deleted. It's not worth continuing.
return
try:
SVN.Capture(['revert', file_status[1]], cwd=cwd)
except subprocess2.CalledProcessError:
if not os.path.exists(file_path):
continue
raise
def RunAndGetFileList(verbose, args, cwd, file_list, stdout=None):
"""Runs svn checkout, update, or status, output to stdout.
The first item in args must be either "checkout", "update", or "status".
svn's stdout is parsed to collect a list of files checked out or updated.
These files are appended to file_list. svn's stdout is also printed to
sys.stdout as in Run.
Args:
verbose: If True, uses verbose output
args: A sequence of command line parameters to be passed to svn.
cwd: The directory where svn is to be run.
Raises:
Error: An error occurred while running the svn command.
"""
stdout = stdout or sys.stdout
# svn update and svn checkout use the same pattern: the first three columns
# are for file status, property status, and lock status. This is followed
# by two spaces, and then the path to the file.
update_pattern = '^... (.*)$'
# The first three columns of svn status are the same as for svn update and
# svn checkout. The next three columns indicate addition-with-history,
# switch, and remote lock status. This is followed by one space, and then
# the path to the file.
status_pattern = '^...... (.*)$'
# args[0] must be a supported command. This will blow up if it's something
# else, which is good. Note that the patterns are only effective when
# these commands are used in their ordinary forms, the patterns are invalid
# for "svn status --show-updates", for example.
pattern = {
'checkout': update_pattern,
'status': status_pattern,
'update': update_pattern,
}[args[0]]
compiled_pattern = re.compile(pattern)
# Place an upper limit.
backoff_time = 5
retries = 0
while True:
retries += 1
previous_list_len = len(file_list)
failure = []
def CaptureMatchingLines(line):
match = compiled_pattern.search(line)
if match:
file_list.append(match.group(1))
if line.startswith('svn: '):
failure.append(line)
try:
gclient_utils.CheckCallAndFilterAndHeader(
['svn'] + args,
cwd=cwd,
always=verbose,
filter_fn=CaptureMatchingLines,
stdout=stdout)
except subprocess2.CalledProcessError:
def IsKnownFailure():
for x in failure:
if (x.startswith('svn: OPTIONS of')
or x.startswith('svn: PROPFIND of')
or x.startswith('svn: REPORT of')
or x.startswith('svn: Unknown hostname')
or x.startswith(
'svn: Server sent unexpected return value')
):
return True
return False
# Subversion client is really misbehaving with Google Code.
if args[0] == 'checkout':
# Ensure at least one file was checked out, otherwise *delete* the
# directory.
if len(file_list) == previous_list_len:
if not IsKnownFailure():
# No known svn error was found, bail out.
raise
# No file were checked out, so make sure the directory is
# deleted in case it's messed up and try again.
# Warning: It's bad, it assumes args[2] is the directory
# argument.
if os.path.isdir(args[2]):
gclient_utils.RemoveDirectory(args[2])
else:
# Progress was made, convert to update since an aborted checkout
# is now an update.
args = ['update'] + args[1:]
else:
# It was an update or export.
# We enforce that some progress has been made or a known failure.
if len(file_list
) == previous_list_len and not IsKnownFailure():
# No known svn error was found and no progress, bail out.
raise
if retries == 10:
raise
print "Sleeping %.1f seconds and retrying...." % backoff_time
time.sleep(backoff_time)
backoff_time *= 1.3
continue
break
SVN._DiffItemInternal(filename,
data[filename],
bogus_dir,
full_move=full_move,
revision=revision))
# Use StringIO since it can be messy when diffing a directory move with
# full_move=True.
buf = cStringIO.StringIO()
for d in filter(None, diffs):
buf.write(d)
result = buf.getvalue()
buf.close()
return result
finally:
os.chdir(previous_cwd)
gclient_utils.RemoveDirectory(bogus_dir)
@staticmethod
def GetEmail(repo_root):
"""Retrieves the svn account which we assume is an email address."""
try:
infos = SVN.CaptureInfo(repo_root)
except subprocess2.CalledProcessError:
return None
# Should check for uuid but it is incorrectly saved for https creds.
root = infos['Repository Root']
realm = root.rsplit('/', 1)[0]
uuid = infos['UUID']
if root.startswith('https') or not uuid:
regexp = re.compile(r'<%s:\d+>.*' % realm)
class SVNWrapper(SCMWrapper):
""" Wrapper for SVN """
def GetRevisionDate(self, revision):
"""Returns the given revision's date in ISO-8601 format (which contains the
time zone)."""
date = scm.SVN.Capture([
'propget', '--revprop', 'svn:date', '-r', revision,
os.path.join(self.checkout_path, '.')
])
return date.strip()
def cleanup(self, options, args, file_list):
"""Cleanup working copy."""
self._Run(['cleanup'] + args, options)
def diff(self, options, args, file_list):
# NOTE: This function does not currently modify file_list.
if not os.path.isdir(self.checkout_path):
raise gclient_utils.Error('Directory %s is not present.' %
self.checkout_path)
self._Run(['diff'] + args, options)
def pack(self, options, args, file_list):
"""Generates a patch file which can be applied to the root of the
repository."""
if not os.path.isdir(self.checkout_path):
raise gclient_utils.Error('Directory %s is not present.' %
self.checkout_path)
gclient_utils.CheckCallAndFilter(
['svn', 'diff', '-x', '--ignore-eol-style'] + args,
cwd=self.checkout_path,
print_stdout=False,
filter_fn=DiffFilterer(self.relpath).Filter)
def update(self, options, args, file_list):
"""Runs svn to update or transparently checkout the working copy.
All updated files will be appended to file_list.
Raises:
Error: if can't get URL for relative path.
"""
# Only update if git or hg is not controlling the directory.
git_path = os.path.join(self.checkout_path, '.git')
if os.path.exists(git_path):
print('________ found .git directory; skipping %s' % self.relpath)
return
hg_path = os.path.join(self.checkout_path, '.hg')
if os.path.exists(hg_path):
print('________ found .hg directory; skipping %s' % self.relpath)
return
if args:
raise gclient_utils.Error("Unsupported argument(s): %s" %
",".join(args))
# revision is the revision to match. It is None if no revision is specified,
# i.e. the 'deps ain't pinned'.
url, revision = gclient_utils.SplitUrlRevision(self.url)
# Keep the original unpinned url for reference in case the repo is switched.
base_url = url
managed = True
if options.revision:
# Override the revision number.
revision = str(options.revision)
if revision:
if revision != 'unmanaged':
forced_revision = True
# Reconstruct the url.
url = '%s@%s' % (url, revision)
rev_str = ' at %s' % revision
else:
managed = False
revision = None
else:
forced_revision = False
rev_str = ''
if not os.path.exists(self.checkout_path):
# We need to checkout.
command = ['checkout', url, self.checkout_path]
command = self._AddAdditionalUpdateFlags(command, options,
revision)
self._RunAndGetFileList(command, options, file_list,
self._root_dir)
return
if not managed:
print('________ unmanaged solution; skipping %s' % self.relpath)
return
# Get the existing scm url and the revision number of the current checkout.
try:
from_info = scm.SVN.CaptureInfo(
os.path.join(self.checkout_path, '.'))
except (gclient_utils.Error, subprocess2.CalledProcessError):
raise gclient_utils.Error((
'Can\'t update/checkout %s if an unversioned directory is present. '
'Delete the directory and try again.') % self.checkout_path)
if 'URL' not in from_info:
raise gclient_utils.Error(
('gclient is confused. Couldn\'t get the url for %s.\n'
'Try using @unmanaged.\n%s') %
(self.checkout_path, from_info))
# Look for locked directories.
dir_info = scm.SVN.CaptureStatus(os.path.join(self.checkout_path, '.'))
if any(d[0][2] == 'L' for d in dir_info):
try:
self._Run(['cleanup', self.checkout_path], options)
except subprocess2.CalledProcessError, e:
# Get the status again, svn cleanup may have cleaned up at least
# something.
dir_info = scm.SVN.CaptureStatus(
os.path.join(self.checkout_path, '.'))
# Try to fix the failures by removing troublesome files.
for d in dir_info:
if d[0][2] == 'L':
if d[0][0] == '!' and options.force:
print 'Removing troublesome path %s' % d[1]
gclient_utils.rmtree(d[1])
else:
print 'Not removing troublesome path %s automatically.' % d[
1]
if d[0][0] == '!':
print 'You can pass --force to enable automatic removal.'
raise e
# Retrieve the current HEAD version because svn is slow at null updates.
if options.manually_grab_svn_rev and not revision:
from_info_live = scm.SVN.CaptureInfo(from_info['URL'])
revision = str(from_info_live['Revision'])
rev_str = ' at %s' % revision
if from_info['URL'] != base_url:
# The repository url changed, need to switch.
try:
to_info = scm.SVN.CaptureInfo(url)
except (gclient_utils.Error, subprocess2.CalledProcessError):
# The url is invalid or the server is not accessible, it's safer to bail
# out right now.
raise gclient_utils.Error('This url is unreachable: %s' % url)
can_switch = (
(from_info['Repository Root'] != to_info['Repository Root'])
and (from_info['UUID'] == to_info['UUID']))
if can_switch:
print('\n_____ relocating %s to a new checkout' % self.relpath)
# We have different roots, so check if we can switch --relocate.
# Subversion only permits this if the repository UUIDs match.
# Perform the switch --relocate, then rewrite the from_url
# to reflect where we "are now." (This is the same way that
# Subversion itself handles the metadata when switch --relocate
# is used.) This makes the checks below for whether we
# can update to a revision or have to switch to a different
# branch work as expected.
# TODO(maruel): TEST ME !
command = [
'switch', '--relocate', from_info['Repository Root'],
to_info['Repository Root'], self.relpath
]
self._Run(command, options, cwd=self._root_dir)
from_info['URL'] = from_info['URL'].replace(
from_info['Repository Root'], to_info['Repository Root'])
else:
if not options.force and not options.reset:
# Look for local modifications but ignore unversioned files.
for status in scm.SVN.CaptureStatus(self.checkout_path):
if status[0] != '?':
raise gclient_utils.Error((
'Can\'t switch the checkout to %s; UUID don\'t match and '
'there is local changes in %s. Delete the directory and '
'try again.') % (url, self.checkout_path))
# Ok delete it.
print('\n_____ switching %s to a new checkout' % self.relpath)
gclient_utils.RemoveDirectory(self.checkout_path)
# We need to checkout.
command = ['checkout', url, self.checkout_path]
command = self._AddAdditionalUpdateFlags(
command, options, revision)
self._RunAndGetFileList(command, options, file_list,
self._root_dir)
return
# If the provided url has a revision number that matches the revision
# number of the existing directory, then we don't need to bother updating.
if not options.force and str(from_info['Revision']) == revision:
if options.verbose or not forced_revision:
print('\n_____ %s%s' % (self.relpath, rev_str))
return
command = ['update', self.checkout_path]
command = self._AddAdditionalUpdateFlags(command, options, revision)
self._RunAndGetFileList(command, options, file_list, self._root_dir)
self._root_dir,
entry_fixed)
scm.status(self._options, [], file_list)
modified_files = file_list != []
if not self._options.delete_unversioned_trees or modified_files:
# There are modified files in this entry. Keep warning until
# removed.
print((
"\nWARNING: \"%s\" is no longer part of this client. "
"It is recommended that you manually remove it.\n")
% entry_fixed)
else:
# Delete the entry
print("\n________ deleting \'%s\' " +
"in \'%s\'") % (entry_fixed, self._root_dir)
gclient_utils.RemoveDirectory(e_dir)
# record the current list of entries for next time
self._SaveEntries(entries)
def PrintRevInfo(self):
"""Output revision info mapping for the client and its dependencies. This
allows the capture of a overall "revision" for the source tree that can
be used to reproduce the same tree in the future. The actual output
contains enough information (source paths, svn server urls and revisions)
that it can be used either to generate external svn commands (without
gclient) or as input to gclient's --rev option (with some massaging of
the data).
NOTE: Unlike RunOnDeps this does not require a local checkout and is run
on the Pulse master. It MUST NOT execute hooks.
