def setupTagsDirectory(self):
if self._setupTagsDirectory == None:
self._setupTagsDirectory = False
if self.module and self.module <> '/':
# Check the existing tags directory
cmd = self.command("ls")
svnls = ExternalCommand(command=cmd)
svnls.execute(self.repository + self.tags_path)
if svnls.exit_status:
# create it, if not exist
cmd = self.command("mkdir", "-m",
"This directory will host the tags")
svnmkdir = ExternalCommand(command=cmd)
svnmkdir.execute(self.repository + self.tags_path)
if svnmkdir.exit_status:
raise TargetInitializationFailure(
"Was not able to create tags directory '%s'" %
self.tags_path)
else:
self.log.debug("Directory '%s' already exists" %
self.tags_path)
self._setupTagsDirectory = True
else:
self.log.debug("Tags needs module setup other than '/'")
return self._setupTagsDirectory
def _prepareWorkingDirectory(self, source_repo):
"""
Checkout a working copy of the target SVN repository.
"""
from os.path import join, exists
from vcpx.dualwd import IGNORED_METADIRS
if not self.repository.repository or exists(
join(self.repository.basedir, self.repository.METADIR)):
return
cmd = self.repository.command("co", "--quiet")
if self.repository.ignore_externals:
cmd.append("--ignore-externals")
svnco = ExternalCommand(command=cmd)
svnco.execute(
"%s%s" % (self.repository.repository, self.repository.module),
self.repository.basedir)
ignore = [md for md in IGNORED_METADIRS]
if self.logfile.startswith(self.repository.basedir):
ignore.append(self.logfile[len(self.repository.basedir) + 1:])
if self.state_file.filename.startswith(self.repository.basedir):
sfrelname = self.state_file.filename[len(self.repository.basedir) +
1:]
ignore.append(sfrelname)
ignore.append(sfrelname + '.old')
ignore.append(sfrelname + '.journal')
cmd = self.repository.command("propset", "%(propname)s", "--quiet")
pset = ExternalCommand(cwd=self.repository.basedir, command=cmd)
pset.execute('\n'.join(ignore), '.', propname='svn:ignore')
def _commit(self,
date,
author,
patchname,
changelog=None,
entries=None,
tags=[],
isinitialcommit=False):
"""
Commit the changeset.
"""
encode = self.repository.encode
logmessage = []
if patchname:
logmessage.append(patchname)
if changelog:
logmessage.append(changelog)
# If we cannot use propset, fall back to old behaviour of
# appending these info to the changelog
if not self.repository.use_propset:
logmessage.append('')
logmessage.append('Original author: %s' % encode(author))
logmessage.append('Date: %s' % date)
elif not self.repository.propset_date:
logmessage.append('')
logmessage.append('Date: %s' % date)
rontf = ReopenableNamedTemporaryFile('svn', 'tailor')
log = open(rontf.name, "w")
log.write(encode('\n'.join(logmessage)))
log.close()
cmd = self.repository.command("commit", "--file", rontf.name)
commit = ExternalCommand(cwd=self.repository.basedir, command=cmd)
if not entries or self.repository.commit_all_files:
entries = ['.']
out, err = commit.execute(entries, stdout=PIPE, stderr=PIPE)
if commit.exit_status:
raise ChangesetApplicationFailure(
"%s returned status %d saying\n%s" %
(str(commit), commit.exit_status, err.read()))
revision = self._propsetRevision(out, commit, date, author)
if not revision:
# svn did not find anything to commit
return
cmd = self.repository.command("update", "--quiet")
if self.repository.ignore_externals:
cmd.append("--ignore-externals")
cmd.extend(["--revision", revision])
ExternalCommand(cwd=self.repository.basedir, command=cmd).execute()
def testExitStatusForFalse(self):
"""Verify ExternalCommand exit_status of ``false``.
"""
if platform != 'win32':
c = ExternalCommand(['false'])
else:
c = ExternalCommand(['cmd', '/c exit 1'])
c.execute()
self.assertNotEqual(c.exit_status, 0)
def testExitStatusForTrue(self):
"""Verify ExternalCommand exit_status of ``true``.
"""
if platform != 'win32':
c = ExternalCommand(['true'])
else:
c = ExternalCommand(['cmd', '/c exit 0'])
c.execute()
self.assertEqual(c.exit_status, 0)
def testStringification(self):
"""Verify the conversion from sequence of args to string"""
c = ExternalCommand(['some spaces here'])
self.assertEqual(str(c), '$ "some spaces here"')
c = ExternalCommand(['a "double quoted" arg'])
self.assertEqual(str(c), r'$ "a \"double quoted\" arg"')
c = ExternalCommand([r'a \" backslashed quote mark\\'])
self.assertEqual(str(c), r'$ "a \\\" backslashed quote mark\\\\"')
def testWorkingDir(self):
"""Verify that the given command is executed in the specified
working directory.
"""
tempdir = gettempdir()
if platform != 'win32':
c = ExternalCommand(['pwd'], tempdir)
else:
c = ExternalCommand(['cmd', '/c', 'cd'], tempdir)
out = c.execute(stdout=PIPE)[0]
self.assertEqual(out.read(), tempdir + "\n")
def create(self):
"""
Create a new monotone DB, storing the commit keys, if available
"""
if not self.repository or exists(self.repository):
return
cmd = self.command("db", "init", "--db", self.repository)
init = ExternalCommand(command=cmd)
init.execute(stdout=PIPE, stderr=PIPE)
if init.exit_status:
raise TargetInitializationFailure("Was not able to initialize "
"the monotone db at %r" %
self.repository)
if self.keyid:
self.log.info("Using key %s for commits" % (self.keyid, ))
else:
# keystore key id unspecified, look at other options
if self.keygenid:
keyfile = join(getenv("HOME"), '.monotone', 'keys',
self.keygenid)
if exists(keyfile):
self.log.info("Key %s exist, don't genkey again" %
self.keygenid)
else:
# requested a new key
cmd = self.command("genkey", "--db", self.repository)
regkey = ExternalCommand(command=cmd)
if self.passphrase:
passp = "%s\n%s\n" % (self.passphrase, self.passphrase)
else:
passp = None
regkey.execute(self.keygenid,
input=passp,
stdout=PIPE,
stderr=PIPE)
if regkey.exit_status:
raise TargetInitializationFailure(
"Was not able to setup "
"the monotone initial key at %r" % self.repository)
else:
raise TargetInitializationFailure("Can't setup the monotone "
"repository %r. "
"A keyid or keygenid "
"must be provided." %
self.repository)
def importFirstRevision(self, source_repo, changeset, initial):
from os import walk, sep
from vcpx.dualwd import IGNORED_METADIRS
if not self.repository.split_initial_import_level:
super(DarcsTargetWorkingDir,
self).importFirstRevision(source_repo, changeset, initial)
else:
cmd = self.repository.command("add", "--case-ok", "--quiet")
add = ExternalCommand(cwd=self.repository.basedir, command=cmd)
cmd = self.repository.command("add", "--case-ok", "--recursive",
"--quiet")
addrecurs = ExternalCommand(cwd=self.repository.basedir,
command=cmd)
for root, dirs, files in walk(self.repository.basedir):
subtree = root[len(self.repository.basedir) + 1:]
if subtree:
log = "Import of subtree %s" % subtree
level = len(subtree.split(sep))
else:
log = "Import of first level"
level = 0
for excd in IGNORED_METADIRS:
if excd in dirs:
dirs.remove(excd)
if level > self.repository.split_initial_import_level:
while dirs:
d = dirs.pop(0)
addrecurs.execute(join(subtree, d))
filenames = [join(subtree, f) for f in files]
if filenames:
add.execute(*filenames)
else:
dirnames = [join(subtree, d) for d in dirs]
if dirnames:
add.execute(*dirnames)
filenames = [join(subtree, f) for f in files]
if filenames:
add.execute(*filenames)
self._commit(changeset.date,
"tailor",
"Initial import",
log,
isinitialcommit=initial)
cmd = self.repository.command("tag", "--author", "tailor")
ExternalCommand(cwd=self.repository.basedir, command=cmd).execute(
"Initial import from %s" % source_repo.repository)
def _getUpstreamChangesets(self, sincerev=None):
from os.path import join, exists
branch = "HEAD"
fname = join(self.repository.basedir, 'CVS', 'Tag')
if exists(fname):
tag = open(fname).read()
if tag.startswith('T'):
branch = tag[1:-1]
else:
if sincerev is not None and isinstance(sincerev, basestring) \
and not sincerev[0] in '0123456789':
branch = sincerev
sincerev = None
if sincerev:
sincerev = int(sincerev)
cmd = self.repository.command("--norc",
"--cvs-direct",
"-u",
"-b",
branch,
"--root",
self.repository.repository,
cvsps=True)
cvsps = ExternalCommand(command=cmd)
log = cvsps.execute(self.repository.module, stdout=PIPE, TZ='UTC0')[0]
for cs in changesets_from_cvsps(log, sincerev):
yield cs
def _removePathnames(self, names):
"""
Remove some filesystem object.
"""
cmd = self.repository.command("remove")
ExternalCommand(cwd=self.repository.basedir, command=cmd).execute(names)
def _commit(self, date, author, patchname, changelog=None, entries=None,
tags = [], isinitialcommit = False):
"""
Commit the changeset.
"""
encode = self.repository.encode
logmessage = []
if patchname:
logmessage.append(patchname)
if changelog:
logmessage.append(changelog.replace('%', '%%'))
cmd = self.repository.command("-u", encode(author), "commit",
"-m", encode('\n'.join(logmessage)),
"-D", date.astimezone(UTC).strftime('%Y/%m/%d %H:%M:%S UTC'))
if not entries:
entries = ['...']
c = ExternalCommand(cwd=self.repository.basedir, command=cmd)
c.execute(entries)
if c.exit_status:
raise ChangesetApplicationFailure("%s returned status %d" %
(str(c), c.exit_status))
def _addPathnames(self, names):
"""
Add some new filesystem objects.
"""
cmd = self.repository.command("add")
ExternalCommand(cwd=self.repository.basedir, command=cmd).execute(names)
def _renamePathname(self, oldname, newname):
"""
Rename a filesystem object.
"""
cmd = self.repository.command("rename")
ExternalCommand(cwd=self.repository.basedir, command=cmd).execute(oldname, newname)
def _tag(self, tagname, date, author):
"""
Apply a tag.
"""
# Sanitize tagnames for CVS: start with [a-zA-z], only include letters,
# numbers, '-' and '_'.
# str.isalpha et al are locale-dependent
def iscvsalpha(chr):
return (chr >= 'a' and chr <= 'z') or (chr >= 'A' and chr <= 'Z')
def iscvsdigit(chr):
return chr >= '0' and chr <= '9'
def iscvschar(chr):
return iscvsalpha(chr) or iscvsdigit(
chr) or chr == '-' or chr == '_'
def cvstagify(chr):
if iscvschar(chr):
return chr
else:
return '_'
tagname = ''.join([cvstagify(chr) for chr in tagname])
if not iscvsalpha(tagname[0]):
tagname = 'tag-' + tagname
cmd = self.repository.command("-f", "tag")
c = ExternalCommand(cwd=self.repository.basedir, command=cmd)
c.execute(tagname)
if c.exit_status:
raise ChangesetApplicationFailure("%s returned status %d" %
(str(c), c.exit_status))
def testDarcsRenameDeleteDir(self):
"Test if darcs to svn fails on moves combined with directory deletes"
from os import mkdir
from os.path import join
from vcpx.tests import DEBUG
drepo = join(self.TESTDIR, 'rename_delete_dir')
mkdir(drepo)
darcs = ExternalCommand(command=['darcs'], cwd=drepo, nolog=not DEBUG)
darcs.execute('init')
dir = join(drepo, 'dir')
mkdir(dir)
darcs.execute('add', dir)
fileA = join(dir, 'fileA')
open(fileA, 'w')
darcs.execute('add', fileA)
darcs.execute('record', '-a', '-m', 'Add dir and dir/A')
fileB = join(drepo, 'fileA')
darcs.execute('rename', fileA, fileB)
darcs.execute('remove', dir)
darcs.execute('record', '-a', '-m', 'Move dir/A to A and delete dir')
self.tailorize('darcs_rename_delete_dir')
def testStandardOutput(self):
"""Verify that ExternalCommand redirects stdout."""
if platform != 'win32':
c = ExternalCommand(['echo'])
else:
c = ExternalCommand(['cmd', '/c', 'echo'])
out = c.execute("ciao", stdout=PIPE)[0]
self.assertEqual(out.read(), "ciao\n")
if platform != 'win32':
out = c.execute('-n', stdout=PIPE)[0]
self.assertEqual(out.read(), '')
out = c.execute("ciao")[0]
self.assertEqual(out, None)
def __move_file(self, old_name, new_name):
#
# The aegis command to rename files does not have the -keep
# option to preserve the content of the file, do it manually.
#
fp = open(os.path.join(self.repository.basedir, new_name), 'rb')
content = fp.read()
fp.close()
cmd = self.repository.command("-move", "-not-logging", "-project",
self.repository.module, "-change",
self.change_number)
move_file = ExternalCommand(cwd=self.repository.basedir, command=cmd)
output = move_file.execute(old_name,
new_name,
stdout=PIPE,
stderr=STDOUT)[0]
if move_file.exit_status > 0:
raise ChangesetApplicationFailure(
"%s returned status %d, saying: %s" %
(str(move_file), move_file.exit_status, output.read()))
#
# Restore the previously saved content of the renamed file.
#
fp = open(os.path.join(self.repository.basedir, new_name), 'wb')
fp.write(content)
fp.close()
def _addPathnames(self, names):
"""
Add some new filesystem objects, skipping directories.
In monotone *explicit* directory addition is always recursive,
so adding a directory here might interfere with renames.
Adding files without directories doesn't cause problems,
because adding a file implicitly adds the parent directory
(non-recursively).
"""
fnames = []
for fn in names:
if isdir(join(self.repository.basedir, fn)):
self.log.debug(
"ignoring addition of directory %r "
"(dirs are implicitly added by files)", fn)
else:
fnames.append(fn)
if len(fnames):
# ok, we still have something to add
cmd = self.repository.command("add", "--")
add = ExternalCommand(cwd=self.repository.basedir, command=cmd)
add.execute(fnames, stdout=PIPE, stderr=PIPE)
if add.exit_status:
raise ChangesetApplicationFailure("%s returned status %s" %
(str(add), add.exit_status))
def _commit(self,
date,
author,
patchname,
changelog=None,
entries=None,
tags=[],
isinitialcommit=False):
"""
Commit the changeset.
"""
logmessage = []
logmessage.append(
date.astimezone(UTC).strftime('%Y/%m/%d %H:%M:%S UTC'))
logmessage.append(author)
if patchname:
logmessage.append(patchname)
if changelog:
logmessage.append(changelog)
if not patchname and not changelog:
logmessage.append('Unnamed patch')
cmd = self.repository.command("record", "--all", "--pipe")
if not entries:
entries = ['.']
record = ExternalCommand(cwd=self.repository.basedir, command=cmd)
record.execute(input=self.repository.encode('\n'.join(logmessage)))
if record.exit_status:
raise ChangesetApplicationFailure(
"%s returned status %d" % (str(record), record.exit_status))
def _commit(self,
date,
author,
patchname,
changelog=None,
entries=None,
tags=[],
isinitialcommit=False):
"""
Commit the changeset.
"""
encode = self.repository.encode
logmessage = []
if patchname:
logmessage.append(patchname)
if changelog:
logmessage.append(changelog.replace('%', '%%'))
cmd = self.repository.command("commit", "-s",
encode('\n'.join(logmessage)),
"--author", encode(author), "--date",
date.isoformat())
c = ExternalCommand(cwd=self.repository.basedir, command=cmd)
c.execute()
if c.exit_status:
raise ChangesetApplicationFailure("%s returned status %d" %
(str(c), c.exit_status))
def _handleConflict(self, changeset, conflicts, conflict):
"""
Handle the conflict raised by the application of the upstream changeset.
Override parent behaviour: with darcs, we need to execute a revert
on the conflicted files, **trashing** local changes, but there should
be none of them in tailor context.
"""
from os import walk, unlink
from os.path import join
from re import compile
self.log.info("Reverting changes to %s, to solve the conflict",
' '.join(conflict))
cmd = self.repository.command("revert", "--all")
revert = ExternalCommand(cwd=self.repository.basedir, command=cmd)
revert.execute(conflict, input="\n")
# Remove also the backups made by darcs
bckre = compile('-darcs-backup[0-9]+$')
for root, dirs, files in walk(self.repository.basedir):
backups = [f for f in files if bckre.search(f)]
for bck in backups:
self.log.debug("Removing backup file %r in %r", bck, root)
unlink(join(root, bck))
def testDarcsRenameDelete(self):
"Try to migrate a darcs patch that renames and removes the same file"
from os import mkdir
from os.path import join
from vcpx.tests import DEBUG
drepo = join(self.TESTDIR, 'rename_delete')
mkdir(drepo)
darcs = ExternalCommand(command=['darcs'], cwd=drepo, nolog=not DEBUG)
darcs.execute('init')
fileA = join(drepo, 'fileA')
open(fileA, 'w')
darcs.execute('add', fileA)
darcs.execute('record', '-a', '-m', 'Add A')
fileB = join(drepo, 'fileB')
darcs.execute('mv', fileA, fileB)
darcs.execute('remove', fileB)
darcs.execute('record', '-a', '-m', 'Move A to B and delete B')
self.tailorize('darcs_rename_delete')
def _getUpstreamChangesets(self, sincerev=None):
if sincerev:
sincerev = int(sincerev)
else:
sincerev = 0
cmd = self.repository.command("log", "--verbose", "--xml",
"--non-interactive", "--revision",
"%d:HEAD" % (sincerev + 1))
svnlog = ExternalCommand(cwd=self.repository.basedir, command=cmd)
log = svnlog.execute('.', stdout=PIPE, TZ='UTC0')[0]
if svnlog.exit_status:
return []
if self.repository.filter_badchars:
from string import maketrans
from cStringIO import StringIO
# Apparently some (SVN repo contains)/(SVN server dumps) some
# characters that are illegal in an XML stream. This was the case
# with Twisted Matrix master repository. To be safe, we replace
# all of them with a question mark.
if isinstance(self.repository.filter_badchars, basestring):
allbadchars = self.repository.filter_badchars
else:
allbadchars = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09" \
"\x0B\x0C\x0E\x0F\x10\x11\x12\x13\x14\x15" \
"\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F\x7f"
tt = maketrans(allbadchars, "?" * len(allbadchars))
log = StringIO(log.read().translate(tt))
return changesets_from_svnlog(log, self.repository)
def __parse_revision_logs(self, fqrevlist, update=True):
changesets = []
logparser = Parser()
c = ExternalCommand(cwd=self.repository.basedir,
command=self.repository.command("cat-archive-log"))
for fqrev in fqrevlist:
out, err = c.execute(fqrev, stdout=PIPE, stderr=PIPE)
if c.exit_status:
raise GetUpstreamChangesetsFailure(
"%s returned status %d saying\n%s" %
(str(c), c.exit_status, err.read()))
err = None
try:
msg = logparser.parse(out)
except Exception, err:
pass
if not err and msg.is_multipart():
err = "unable to parse log description"
if not err and update and msg.has_key('Continuation-of'):
err = "in-version continuations not supported"
if err:
raise GetUpstreamChangesetsFailure(str(err))
date = self.__parse_date(msg['Date'], msg['Standard-date'])
author = msg['Creator']
revision = fqrev
logmsg = [msg['Summary']]
s = msg.get('Keywords', "").strip()
if s:
logmsg.append('Keywords: ' + s)
s = msg.get_payload().strip()
if s:
logmsg.append(s)
logmsg = '\n'.join(logmsg)
changesets.append(Changeset(revision, date, author, logmsg))
def _removePathnames(self, names):
"""
Remove some filesystem objects.
"""
cmd = self.repository.command("remove", "--quiet", "--force")
remove = ExternalCommand(cwd=self.repository.basedir, command=cmd)
remove.execute(names)
def __delta_name(self, delta):
cmd = self.repository.command("-delta_name", "-project",
self.repository.module)
delta_name = ExternalCommand(cwd="/tmp", command=cmd)
output = delta_name.execute(delta, stdout=PIPE, stderr=STDOUT)[0]
if delta_name.exit_status > 0:
raise ChangesetApplicationFailure(
"%s returned status %d, saying: %s" %
(str(delta_name), delta_name.exit_status, output.read()))
def _addPathnames(self, names):
"""
Add some new filesystem objects.
"""
cmd = self.repository.command("add", "--quiet", "--no-auto-props",
"--non-recursive")
ExternalCommand(cwd=self.repository.basedir,
command=cmd).execute(names)
def darcs_version(self):
if self._darcs_version is None:
cmd = self.command('--version')
version = ExternalCommand(command=cmd)
self._darcs_version = version.execute(
stdout=PIPE)[0].read().strip()
self.log.debug('Using %s, version %s', self.EXECUTABLE,
self._darcs_version)
return self._darcs_version
def __apply_changeset(self, changeset):
c = ExternalCommand(cwd=self.repository.basedir,
command=self.repository.command("update"))
out, err = c.execute(changeset.revision, stdout=PIPE, stderr=PIPE)
if not c.exit_status in [0, 1]:
raise ChangesetApplicationFailure(
"%s returned status %d saying\n%s" %
(str(c), c.exit_status, err.read()))
return self.__parse_apply_changeset_output(changeset, out)
