def call_command(command, **kw):
"""Call the specified command, checking that it exits successfully.
Raise a FatalError if the command cannot be executed, or if it exits
with a non-zero exit code. Pass KW as keyword arguments to
subprocess.call()."""
logger.debug('Running command %r' % (command,))
try:
retcode = subprocess.call(command, **kw)
if retcode < 0:
raise FatalError(
'Command terminated by signal %d: "%s"'
% (-retcode, ' '.join(command),)
)
elif retcode > 0:
raise FatalError(
'Command failed with return code %d: "%s"'
% (retcode, ' '.join(command),)
)
except OSError, e:
raise FatalError(
'Command execution failed (%s): "%s"'
% (e, ' '.join(command),)
)
def check(self):
if Ctx().cross_project_commits:
raise FatalError(
'Git output is not supported with cross-project commits')
if Ctx().cross_branch_commits:
raise FatalError(
'Git output is not supported with cross-branch commits')
if Ctx().username is None:
raise FatalError('Git output requires a default commit username')
def __init__(
self,
id,
project_cvs_repos_path,
initial_directories=[],
symbol_transforms=None,
exclude_paths=[],
):
"""Create a new Project record.
ID is a unique id for this project. PROJECT_CVS_REPOS_PATH is the
main CVS directory for this project (within the filesystem).
INITIAL_DIRECTORIES is an iterable of all SVN directories that
should be created when the project is first created. Normally,
this should include the trunk, branches, and tags directory.
SYMBOL_TRANSFORMS is an iterable of SymbolTransform instances
which will be used to transform any symbol names within this
project.
EXCLUDE_PATHS is an iterable of paths that should be excluded from
the conversion. The paths should be relative to
PROJECT_CVS_REPOS_PATH and use slashes ('/'). Paths for
individual files should include the ',v' extension.
"""
self.id = id
self.project_cvs_repos_path = os.path.normpath(project_cvs_repos_path)
if not os.path.isdir(self.project_cvs_repos_path):
raise FatalError(
"The specified CVS repository path '%s' is not an "
"existing directory." % self.project_cvs_repos_path)
self.cvs_repository_root, self.cvs_module = \
self.determine_repository_root(
os.path.abspath(self.project_cvs_repos_path))
# The SVN directories to add when the project is first created:
self._initial_directories = []
for path in initial_directories:
try:
path = normalize_svn_path(path, False)
except IllegalSVNPathError, e:
raise FatalError(
'Initial directory %r is not a legal SVN path: %s' % (
path,
e,
))
self._initial_directories.append(path)
def __init__(
self,
id,
project_cvs_repos_path,
initial_directories=[],
symbol_transforms=None,
):
"""Create a new Project record.
ID is a unique id for this project. PROJECT_CVS_REPOS_PATH is the
main CVS directory for this project (within the filesystem).
INITIAL_DIRECTORIES is an iterable of all SVN directories that
should be created when the project is first created. Normally,
this should include the trunk, branches, and tags directory.
SYMBOL_TRANSFORMS is an iterable of SymbolTransform instances
which will be used to transform any symbol names within this
project."""
self.id = id
self.project_cvs_repos_path = os.path.normpath(project_cvs_repos_path)
if not os.path.isdir(self.project_cvs_repos_path):
raise FatalError(
"The specified CVS repository path '%s' is not an "
"existing directory." % self.project_cvs_repos_path)
self.cvs_repository_root, self.cvs_module = \
self.determine_repository_root(
os.path.abspath(self.project_cvs_repos_path))
# A regexp matching project_cvs_repos_path plus an optional separator:
self.project_prefix_re = re.compile(
r'^' + re.escape(self.project_cvs_repos_path) + r'(' +
re.escape(os.sep) + r'|$)')
# The SVN directories to add when the project is first created:
self._initial_directories = []
for path in initial_directories:
try:
path = normalize_svn_path(path, False)
except IllegalSVNPathError, e:
raise FatalError(
'Initial directory %r is not a legal SVN path: %s' % (
path,
e,
))
self._initial_directories.append(path)
def main(progname, run_options, pass_manager):
# Convenience var, so we don't have to keep instantiating this Borg.
ctx = Ctx()
# Make sure the tmp directory exists. Note that we don't check if
# it's empty -- we want to be able to use, for example, "." to hold
# tempfiles.
if ctx.tmpdir is None:
ctx.tmpdir = tempfile.mkdtemp(prefix=('%s-' % (progname, )))
erase_tmpdir = True
logger.quiet(
'Writing temporary files to %r\n'
'Be sure to use --tmpdir=%r if you need to resume this conversion.'
% (
ctx.tmpdir,
ctx.tmpdir,
), )
elif not os.path.exists(ctx.tmpdir):
os.mkdir(ctx.tmpdir)
erase_tmpdir = True
elif not os.path.isdir(ctx.tmpdir):
raise FatalError(
"cvs2svn tried to use '%s' for temporary files, but that path\n"
" exists and is not a directory. Please make it be a directory,\n"
" or specify some other directory for temporary files." %
(ctx.tmpdir, ))
else:
erase_tmpdir = False
# But do lock the tmpdir, to avoid process clash.
try:
os.mkdir(os.path.join(ctx.tmpdir, 'cvs2svn.lock'))
except OSError, e:
if e.errno == errno.EACCES:
raise FatalError("Permission denied:" +
" No write access to directory '%s'." %
ctx.tmpdir)
if e.errno == errno.EEXIST:
raise FatalError(
"cvs2svn is using directory '%s' for temporary files, but\n"
" subdirectory '%s/cvs2svn.lock' exists, indicating that another\n"
" cvs2svn process is currently using '%s' as its temporary\n"
" workspace. If you are certain that is not the case,\n"
" then remove the '%s/cvs2svn.lock' subdirectory." % (
ctx.tmpdir,
ctx.tmpdir,
ctx.tmpdir,
ctx.tmpdir,
))
raise
def main(progname, cmd_args):
# Disable garbage collection, as we try not to create any circular
# data structures:
gc.disable()
# Convenience var, so we don't have to keep instantiating this Borg.
ctx = Ctx()
pass_manager = PassManager(passes)
run_options = RunOptions(progname, cmd_args, pass_manager)
# Make sure the tmp directory exists. Note that we don't check if
# it's empty -- we want to be able to use, for example, "." to hold
# tempfiles. But if we *did* want check if it were empty, we'd do
# something like os.stat(ctx.tmpdir)[stat.ST_NLINK], of course :-).
if not os.path.exists(ctx.tmpdir):
erase_tmpdir = True
os.mkdir(ctx.tmpdir)
elif not os.path.isdir(ctx.tmpdir):
raise FatalError(
"cvs2svn tried to use '%s' for temporary files, but that path\n"
" exists and is not a directory. Please make it be a directory,\n"
" or specify some other directory for temporary files." %
(ctx.tmpdir, ))
else:
erase_tmpdir = False
# But do lock the tmpdir, to avoid process clash.
try:
os.mkdir(os.path.join(ctx.tmpdir, 'cvs2svn.lock'))
except OSError, e:
if e.errno == errno.EACCES:
raise FatalError("Permission denied:" +
" No write access to directory '%s'." %
ctx.tmpdir)
if e.errno == errno.EEXIST:
raise FatalError(
"cvs2svn is using directory '%s' for temporary files, but\n"
" subdirectory '%s/cvs2svn.lock' exists, indicating that another\n"
" cvs2svn process is currently using '%s' as its temporary\n"
" workspace. If you are certain that is not the case,\n"
" then remove the '%s/cvs2svn.lock' subdirectory." % (
ctx.tmpdir,
ctx.tmpdir,
ctx.tmpdir,
ctx.tmpdir,
))
raise
def __init__(self, value):
if value not in ['collapsed', 'expanded', 'untouched', None]:
raise FatalError(
'Value for %s must be "collapsed", "expanded", or "untouched"'
% (self.propname,)
)
self.value = value
def _get_cvs_file(
self, parent_directory, basename,
file_in_attic=False, leave_in_attic=False,
):
"""Return a CVSFile describing the file with name BASENAME.
PARENT_DIRECTORY is the CVSDirectory instance describing the
directory that physically holds this file in the filesystem.
BASENAME must be the base name of a *,v file within
PARENT_DIRECTORY.
FILE_IN_ATTIC is a boolean telling whether the specified file is
in an Attic subdirectory. If FILE_IN_ATTIC is True, then:
- If LEAVE_IN_ATTIC is True, then leave the 'Attic' component in
the filename.
- Otherwise, raise FileInAndOutOfAtticException if a file with the
same filename appears outside of Attic.
The CVSFile is assigned a new unique id. All of the CVSFile
information is filled in except mode (which can only be determined
by parsing the file).
Raise FatalError if the resulting filename would not be legal in
SVN."""
filename = os.path.join(parent_directory.rcs_path, basename)
try:
Ctx().output_option.verify_filename_legal(basename[:-2])
except IllegalSVNPathError, e:
raise FatalError(
'File %r would result in an illegal SVN filename: %s'
% (filename, e,)
)
def process_io_options(self):
"""Process input/output options.
Process options related to extracting data from the CVS repository
and writing to a Bazaar-friendly fast-import file."""
ctx = Ctx()
options = self.options
not_both(options.use_rcs, '--use-rcs', options.use_cvs, '--use-cvs')
if options.use_rcs:
revision_reader = RCSRevisionReader(
co_executable=options.co_executable)
else:
# --use-cvs is the default:
revision_reader = CVSRevisionReader(
cvs_executable=options.cvs_executable)
if not ctx.dry_run and not options.dumpfile:
raise FatalError("must pass '--dry-run' or '--dumpfile' option.")
ctx.revision_recorder = NullRevisionRecorder()
ctx.revision_excluder = NullRevisionExcluder()
ctx.revision_reader = None
ctx.output_option = GitOutputOption(
options.dumpfile,
GitRevisionInlineWriter(revision_reader),
max_merges=None,
# Optional map from CVS author names to bzr author names:
author_transforms={}, # FIXME
)
def callback_symbol_transform(self, option, opt_str, value, parser):
[pattern, replacement] = value.split(":")
try:
parser.values.symbol_transforms.append(
RegexpSymbolTransform(pattern, replacement))
except re.error:
raise FatalError("'%s' is not a valid regexp." % (pattern, ))
def get_content(self, cvs_rev):
"""Check out the text for revision C_REV from the repository.
Return the text. If CVS_REV has a property _keyword_handling, use
it to determine how to handle RCS keywords in the output:
'collapsed' -- collapse keywords
'expanded' -- expand keywords
'untouched' -- output keywords in the form they are found in
the RCS file
Note that $Log$ never actually generates a log (which makes test
'requires_cvs()' fail).
Revisions may be requested in any order, but if they are not
requested in dependency order the checkout database will become
very large. Revisions may be skipped. Each revision may be
requested only once."""
try:
text = self._get_text_record(cvs_rev).checkout(
self._text_record_db)
except MalformedDeltaException, (msg):
raise FatalError('Malformed RCS delta in %s, revision %s: %s' %
(cvs_rev.cvs_file.rcs_path, cvs_rev.rev, msg))
def __init__(self, cvs_executable, global_options=None):
"""Initialize a CVSRevisionReader.
CVS_EXECUTABLE is the CVS command (possibly including the full
path to the executable; otherwise it is sought in the $PATH).
GLOBAL_ARGUMENTS, if specified, should be a list of global options
that are passed to the CVS command before the subcommand. If
GLOBAL_ARGUMENTS is not specified, then each of the possibilities
listed in _possible_global_options is checked in order until one
is found that runs successfully and without any output to stderr."""
self.cvs_executable = cvs_executable
if global_options is None:
for global_options in self._possible_global_options:
try:
self._check_cvs_runs(global_options)
except CommandFailedException, e:
pass
else:
break
else:
raise FatalError(
'%s\n'
'Please check that cvs is installed and in your PATH.' %
(e, ))
def determine_repository_root(path):
"""Ascend above the specified PATH if necessary to find the
cvs_repository_root (a directory containing a CVSROOT directory)
and the cvs_module (the path of the conversion root within the cvs
repository). Return the root path and the module path of this
project relative to the root.
NB: cvs_module must be seperated by '/', *not* by os.sep."""
def is_cvs_repository_root(path):
return os.path.isdir(os.path.join(path, 'CVSROOT'))
original_path = path
cvs_module = ''
while not is_cvs_repository_root(path):
# Step up one directory:
prev_path = path
path, module_component = os.path.split(path)
if path == prev_path:
# Hit the root (of the drive, on Windows) without finding a
# CVSROOT dir.
raise FatalError(
"the path '%s' is not a CVS repository, nor a path "
"within a CVS repository. A CVS repository contains "
"a CVSROOT directory within its root directory." %
(original_path, ))
cvs_module = module_component + "/" + cvs_module
return path, cvs_module
def get_content(self, cvs_rev):
# Is EOL fixing requested?
eol_fix = cvs_rev.get_property('_eol_fix') or None
# How do we want keywords to be handled?
keyword_handling = cvs_rev.get_property('_keyword_handling') or None
try:
(k_option,
explicit_keyword_handling) = self._text_options[bool(eol_fix),
keyword_handling]
except KeyError:
raise FatalError(
'Undefined _keyword_handling property (%r) for %s' % (
keyword_handling,
cvs_rev,
))
data = get_command_output(self.get_pipe_command(cvs_rev, k_option))
if Ctx().decode_apple_single:
# Insert a filter to decode any files that are in AppleSingle
# format:
data = get_maybe_apple_single(data)
if explicit_keyword_handling == 'expanded':
data = expand_keywords(data, cvs_rev)
elif explicit_keyword_handling == 'collapsed':
data = collapse_keywords(data)
if eol_fix:
data = canonicalize_eol(data, eol_fix)
return data
def process_output_options(self):
"""Process options related to fastimport output."""
ctx = Ctx()
options = self.options
if options.use_rcs:
revision_reader = RCSRevisionReader(
co_executable=options.co_executable)
else:
# --use-cvs is the default:
revision_reader = CVSRevisionReader(
cvs_executable=options.cvs_executable)
if not ctx.dry_run and not options.dumpfile:
raise FatalError("must pass '--dry-run' or '--dumpfile' option.")
# See cvs2bzr-example.options for explanations of these
ctx.revision_collector = NullRevisionCollector()
ctx.revision_reader = None
if ctx.dry_run:
ctx.output_option = NullOutputOption()
else:
ctx.output_option = BzrOutputOption(
options.dumpfile,
GitRevisionInlineWriter(revision_reader),
# Optional map from CVS author names to bzr author names:
author_transforms={}, # FIXME
)
def write(self, s):
try:
self.loader_pipe.stdin.write(s)
except IOError:
raise FatalError('svnadmin failed with the following output while '
'loading the dumpfile:\n%s' %
(self.loader_pipe.stderr.read(), ))
def compute_best_source(self, preferred_source):
"""Determine the best source_lod and subversion revision number to copy.
Return the best source found, as an SVNRevisionRange instance. If
PREFERRED_SOURCE is not None and its opening is among the sources
with the best scores, return it; otherwise, return the oldest such
revision on the first such source_lod (ordered by the natural LOD
sort order). The return value's source_lod is the best LOD to
copy from, and its opening_revnum is the best SVN revision."""
# Aggregate openings and closings from our rev tree
svn_revision_ranges = self._get_revision_ranges(self._node_tree)
# Score the lists
revision_scores = RevisionScores(svn_revision_ranges)
best_source_lod, best_revnum, best_score = \
revision_scores.get_best_revnum()
if (preferred_source is not None
and revision_scores.get_score(preferred_source) == best_score):
best_source_lod = preferred_source.source_lod
best_revnum = preferred_source.opening_revnum
if best_revnum == SVN_INVALID_REVNUM:
raise FatalError(
"failed to find a revision to copy from when copying %s" %
self._symbol.name)
return SVNRevisionRange(best_source_lod, best_revnum)
def normalize_ttb_path(opt, path, allow_empty=False):
try:
return normalize_svn_path(path, allow_empty)
except IllegalSVNPathError, e:
raise FatalError('Problem with %s: %s' % (
opt,
e,
))
def __init__(self, co_executable):
self.co_executable = co_executable
try:
check_command_runs([self.co_executable, '-V'], self.co_executable)
except CommandFailedException, e:
raise FatalError('%s\n'
'Please check that co is installed and in your PATH\n'
'(it is a part of the RCS software).' % (e,))
def transform(self, cvs_file, symbol_name, revision):
try:
return normalize_svn_path(symbol_name)
except IllegalSVNPathError, e:
raise FatalError('Problem with %s: %s' % (
symbol_name,
e,
))
def process_output_options(self):
"""Process the options related to SVN output."""
ctx = Ctx()
options = self.options
if options.dump_only and not options.dumpfile:
raise FatalError(
"'--dump-only' requires '--dumpfile' to be specified.")
if not options.svnrepos and not options.dumpfile and not ctx.dry_run:
raise FatalError("must pass one of '-s' or '--dumpfile'.")
not_both(options.svnrepos, '-s', options.dumpfile, '--dumpfile')
not_both(options.dumpfile, '--dumpfile', options.existing_svnrepos,
'--existing-svnrepos')
not_both(options.bdb_txn_nosync, '--bdb-txn-nosync',
options.existing_svnrepos, '--existing-svnrepos')
not_both(options.dumpfile, '--dumpfile', options.bdb_txn_nosync,
'--bdb-txn-nosync')
not_both(options.fs_type, '--fs-type', options.existing_svnrepos,
'--existing-svnrepos')
if (options.fs_type and options.fs_type != 'bdb'
and options.bdb_txn_nosync):
raise FatalError(
"cannot pass --bdb-txn-nosync with --fs-type=%s." %
options.fs_type)
if options.svnrepos:
if options.existing_svnrepos:
ctx.output_option = ExistingRepositoryOutputOption(
options.svnrepos)
else:
ctx.output_option = NewRepositoryOutputOption(
options.svnrepos,
fs_type=options.fs_type,
bdb_txn_nosync=options.bdb_txn_nosync,
create_options=options.create_options)
else:
ctx.output_option = DumpfileOutputOption(options.dumpfile)
def finish(self):
self._popen.stdin.close()
logger.normal('Waiting for generate_blobs.py to finish...')
returncode = self._popen.wait()
if returncode:
raise FatalError('generate_blobs.py failed with return code %s.' %
(returncode, ))
else:
logger.normal('generate_blobs.py is done.')
def callback_encoding(self, option, opt_str, value, parser):
ctx = Ctx()
try:
ctx.cvs_author_decoder.add_encoding(value)
ctx.cvs_log_decoder.add_encoding(value)
ctx.cvs_filename_decoder.add_encoding(value)
except LookupError, e:
raise FatalError(str(e))
def callback_fallback_encoding(self, option, opt_str, value, parser):
ctx = Ctx()
try:
ctx.cvs_author_decoder.set_fallback_encoding(value)
ctx.cvs_log_decoder.set_fallback_encoding(value)
# Don't use fallback_encoding for filenames.
except LookupError, e:
raise FatalError(str(e))
def utf8_path(path):
"""Return a copy of PATH encoded in UTF-8."""
try:
return Ctx().cvs_filename_decoder.decode_path(path).encode('utf8')
except UnicodeError:
raise FatalError(
"Unable to convert a path '%s' to internal encoding.\n"
"Consider rerunning with one or more '--encoding' parameters or\n"
"with '--fallback-encoding'." % (path, ))
def check_options(self):
"""Check the the run options are OK.
This should only be called after all options have been processed."""
# Convenience var, so we don't have to keep instantiating this Borg.
ctx = Ctx()
if not self.start_pass <= self.end_pass:
raise InvalidPassError(
'Ending pass must not come before starting pass.')
if not ctx.dry_run and ctx.output_option is None:
raise FatalError('No output option specified.')
if ctx.output_option is not None:
ctx.output_option.check()
if not self.projects:
raise FatalError('No project specified.')
def get_symbol(self, symbol, stats):
if not isinstance(symbol, LineOfDevelopment):
# This symbol will not be included in the conversion; skip it.
return symbol
if symbol.base_path is not None:
# This lod's base path is already set; leave it.
pass
elif isinstance(symbol, Trunk):
trunk_path = symbol.project.trunk_path
if trunk_path is None:
raise FatalError('DefaultBasePathRule used for trunk,\n'
'but project\'s trunk path is not set')
symbol.base_path = trunk_path
elif isinstance(symbol, Branch):
branches_path = symbol.project.branches_path
if branches_path is None:
raise FatalError('DefaultBasePathRule used for symbol %s,\n'
'but project\'s branch path is not set' %
(symbol, ))
symbol.base_path = path_join(branches_path, symbol.name)
elif isinstance(symbol, Tag):
tags_path = symbol.project.tags_path
if tags_path is None:
raise FatalError('DefaultBasePathRule used for symbol %s,\n'
'but project\'s tag path is not set' %
(symbol, ))
symbol.base_path = path_join(tags_path, symbol.name)
else:
raise NotImplementedError()
return symbol
def _utf8_path(self, path):
"""Return a copy of PATH encoded in UTF-8."""
# Convert each path component separately (as they may each use
# different encodings).
try:
return '/'.join([
Ctx().cvs_filename_decoder(piece).encode('utf8')
for piece in path.split('/')
])
except UnicodeError:
raise FatalError(
"Unable to convert a path '%s' to internal encoding.\n"
"Consider rerunning with one or more '--encoding' parameters or\n"
"with '--fallback-encoding'." % (path, ))
def _mutate_branch_to_tag(self, cvs_branch):
"""Mutate the branch CVS_BRANCH into a tag."""
if cvs_branch.next_id is not None:
# This shouldn't happen because it was checked in
# CollateSymbolsPass:
raise FatalError('Attempt to exclude a branch with commits.')
cvs_tag = CVSTag(
cvs_branch.id, cvs_branch.cvs_file, cvs_branch.symbol,
cvs_branch.source_lod, cvs_branch.source_id,
cvs_branch.revision_reader_token,
)
self.add(cvs_tag)
cvs_revision = self[cvs_tag.source_id]
cvs_revision.branch_ids.remove(cvs_tag.id)
cvs_revision.tag_ids.append(cvs_tag.id)
def end_commit(self):
"""Feed the revision stored in the dumpfile to the svnadmin load pipe."""
DumpfileDelegate.end_commit(self)
self.dumpfile.seek(0)
while True:
data = self.dumpfile.read(128 * 1024) # Chunk size is arbitrary
if not data:
break
try:
self.loader_pipe.stdin.write(data)
except IOError:
raise FatalError("svnadmin failed with the following output "
"while loading the dumpfile:\n" +
self.loader_pipe.stderr.read())
self.dumpfile.seek(0)
self.dumpfile.truncate()
def __init__(self, msg):
FatalError.__init__(
self, msg + '\nUse --help-passes for more information.')
