def generate_changesets(self, progress=None, limit=None):
"""Convert changes stored in the meta database into sets of
related changes and store the resulting changesets in the meta
database as well.
"""
if progress == None:
progress = NoProgress()
with progress:
count = 0
csg = ChangeSetGenerator()
total = self.metadb.count_changes()
progress(_('Processing changes'), 0, total)
for change in self.changes(processed=False, reentrant=True):
count += 1
progress(_('Processing changes'), count, total)
for cs in csg.integrate(change):
# XXX: not reflected in progress output, and ugly
if limit:
if limit > 0:
limit -= 1
else:
return
self.metadb.add_changeset(cs)
for cs in csg.finalize():
# XXX: not reflected in progress output, and ugly
if limit:
if limit > 0:
limit -= 1
else:
return
self.metadb.add_changeset(cs)
def _zombie_check(self, result, parent, filename):
"""Check a path for zombie files. If a path exists in the Attic
and the parent directory, one of them must be a zombie copy. If
it cannot be determined which one is the zombie and which one is
the real copy, raise an error.
This function should only be called during an os.walk() run when
searching for files an Attic directory. This guarantees that we
have already seen the other copy in the parent directory, if one
exists. The return value is True if the zombie is in the Attic
and False if the zombie is in the parent directory."""
trunkfile = os.path.join(parent, filename)
if not os.path.isfile(trunkfile):
# No zombie present; the file exists only in Attic.
return False
atticfile = os.path.join(parent, 'Attic', filename)
# FIXME: Not a reliable test. We should make sure that the
# zombie contains a subset of the revisions of the real copy.
if os.path.getsize(trunkfile) < os.path.getsize(atticfile):
result.remove(trunkfile)
return False
raise RuntimeError, \
_("invalid path: %s (%s)") % (trunkfile, \
_('exists in Attic and parent directory'))
def finalize_options(self):
if len(self.args) < 1:
self.usage_error(_('missing RCS file argument'))
elif len(self.args) == 1:
self.rcsfile = self.args[0]
else:
self.usage_error(_('too many arguments'))
def blob(self, change, changeset):
"""Return the raw binary content of a file at the specified
revision.
"""
filename = change.filename + ',v'
rcsfile = os.path.join(self.prefix, filename)
if not os.path.isfile(rcsfile):
rcsfile = os.path.join(self.prefix,
os.path.dirname(filename), 'Attic',
os.path.basename(filename))
if not os.path.isfile(rcsfile):
raise RuntimeError, _('no RCS file found for %s') % filename
# cvs has the odd behavior that it favors revision 1.1 over
# 1.1.1.1 if it searches for a revision by date and the date
# is after that of the initial revision even by one second.
#if change.revision == '1.1.1.1' and \
# change.timestamp < changeset.timestamp:
# revision = '1.1'
#else:
# revision = change.revision
revision = change.revision
rcsfile = RCSFile(rcsfile)
blob = rcsfile.blob(revision)
return self.expand_keywords(blob, change, rcsfile, revision)
def initialize_options(self):
self.add_option('--limit', type='int', metavar='COUNT', help=\
_("Stop importing after COUNT new commits."))
self.add_option('--quiet', action='store_true', help=\
_("Only report error and warning messages."))
self.add_option('--verbose', action='store_true', help=\
_("Display each changeset as it is imported."))
self.add_authors_option()
self.add_stop_on_unknown_author_option()
def initialize_options(self):
self.repository = None
self.add_option('--bare', action='store_true', help=\
_("Initialize a bare repository. See git-init(1)."))
self.add_option('--domain', metavar='DOMAIN', help=\
_("Set the 'cvs.domain' configuration option to the "
"e-mail domain to use as a default value for unknown "
"authors."))
self.add_option('--quiet', action='store_true', help=\
_("Only print error and warning messages."))
def finalize_options(self):
if len(self.args) < 1:
self.usage_error(_('missing CVS repository path'))
elif len(self.args) == 1:
self.repository = self.args[0]
self.directory = None
elif len(self.args) == 2:
self.repository = self.args[0]
self.directory = self.args[1]
else:
self.usage_error(_('too many arguments'))
def initialize_options(self):
self.commit = 'HEAD'
self.add_option('--history', action='store_true', help=\
_("Walk backwards through the entire history."))
self.add_option('--forward', action='store_true', help=\
_("Modifies the --history option to walk forward."))
self.add_option('--skip', action='store_true', help=\
_("Skip the first verification step and move HEAD "
"backward instead, the opposite direction with "
"--forward)."))
self.add_option('--quiet', action='store_true', help=\
_("Only report error and warning messages."))
def finalize_options(self):
if len(self.args) < 1:
self.usage_error(_('missing CVS repository path'))
elif len(self.args) == 1:
self.repository = os.path.abspath(self.args[0])
self.directory = os.path.basename(self.repository)
elif len(self.args) == 2:
self.repository, self.directory = self.args
else:
self.usage_error(_('too many arguments'))
self.finalize_authors_option()
def initialize_options(self):
self.add_option('--limit', type='int', metavar='COUNT', help=\
_("Stop importing after COUNT new commits."))
self.add_option('--quiet', action='store_true', help=\
_("Only report error and warning messages."))
self.add_option('--verbose', action='store_true', help=\
_("Display each changeset as it is imported."))
self.add_option('--verify', action='store_true', help=\
_("Verify the new HEAD revision and work tree against "
"a fresh CVS checkout (does not work in a bare "
"repository.)"))
self.add_authors_option()
self.add_stop_on_unknown_author_option()
def commit_date(self, commit):
"""Get the commit date as a string in UTC timezone.
"""
command = ['git', 'log', '-1', commit]
pipe = self.git._popen(command, stdout=PIPE, stderr=PIPE)
stdout, stderr = pipe.communicate()
if pipe.returncode != 0:
self.fatal(_("can't get commit date of %s" % commit))
match = re.search('Date: +[^ ]+ (.*) \+0000', stdout)
if match:
return '%s UTC' % match.group(1)
else:
self.fatal(_("couldn't match Date: in output of '%s'") % \
' '.join(command))
def initialize_options(self):
self.repository = None
self.directory = None
self.add_option('--bare', action='store_true', help=\
_("Create a bare Git repository without work tree."))
self.add_option('--limit', type='int', metavar='COUNT', help=\
_("Stop importing after COUNT new commits."))
self.add_option('--domain', metavar='DOMAIN', help=\
_("Set the e-mail domain to use for unknown authors."))
self.add_option('--verify', action='store_true', help=\
_("Run the verify command after cloning (does not work "
"with --bare)."))
self.add_quiet_option()
self.add_verbose_option()
self.add_authors_option()
self.add_stop_on_unknown_author_option()
def split_cvs_source(dirname):
"""Split <dirname> into CVSROOT and module paths.
"""
cvsroot = dirname
module = ''
while True:
parent = os.path.dirname(cvsroot)
if cvsroot == parent:
raise TypeError, _('not a CVS repository path (%s): %s') \
% (_('no CVSROOT within nor above'), dirname)
if os.path.isdir(os.path.join(cvsroot, 'CVSROOT')):
return (cvsroot, module,)
if module == '':
module = os.path.basename(cvsroot)
else:
module = os.path.join(os.path.basename(cvsroot), module)
cvsroot = parent
def finalize_options(self):
if len(self.args) > 0:
self.usage_error(_('too many arguments'))
if self.options.quiet:
self.progress = NoProgress()
else:
self.progress = Progress()
def rcsfilename(self, change):
"""Return the RCS filename corresponding to <change>.
"""
filename = change.filename + ',v'
result = os.path.join(self.prefix, filename)
if os.path.isfile(result):
return result
result = os.path.join(self.prefix,
os.path.dirname(filename), 'Attic',
os.path.basename(filename))
if os.path.isfile(result):
return result
raise RuntimeError, _('no RCS file found for %s') % \
change.filename
def _fetch_changes(self, filenames, progress):
count = 0
total = len(filenames)
progress(_('Parsing RCS files'), count, total)
# We will commit changes to the database every few seconds to
# avoid having to scan all RCS files again in case the process
# is interrupted or an error occurs (a rescan isn't really bad
# but costs time.)
commit_time = 0
commit_interval = 10
self.metadb.begin_transaction()
for rcsfile in filenames:
try:
self._fetch_changes_from_rcsfile(rcsfile)
if time.time() - commit_time >= commit_interval:
try:
self.metadb.end_transaction()
except:
pass
self.metadb.begin_transaction()
commit_time = time.time()
count += 1
progress(_('Parsing RCS files'), count, total)
except KeyboardInterrupt:
# Re-raise the exception silently. An impatient user
# may interrupt the process and that should by handled
# gracefully.
raise
except:
# Print the file name where this error happened,
# regardless of whether the error is actually printed,
# just as a quick & dirty debugging aid.
# TODO: raise a FetchChangesError
print "(Error while processing %s)" % rcsfile
raise
finally:
try:
self.metadb.end_transaction()
except:
pass
def __init__(self, rcsfile, revision):
"""This exception provides additional information.
'rcsfile' is an RCSFile object.
'revision' is the revision that couldn't be retrieved.
"""
super(CheckoutError, self).__init__(
_("Couldn't retrieve file content for revision %s of %s") % \
(revision, rcsfile.filename), rcsfile)
self.revision = revision
def _changed_rcs_filenames(self, progress=None):
# Helper function to raise the OSError reported by os.walk().
def raise_error(e): raise e
self.statcache = self.metadb.load_statcache()
result = []
count = 0
for dirpath, dirnames, filenames in \
os.walk(self.prefix, onerror=raise_error):
# Convert from absolute to relative path.
assert(dirpath.startswith(self.prefix))
dirpath = dirpath[len(self.prefix)+1:]
# TODO: fill stat() cache for directories
#if self._unmodified(dirpath):
# continue
# Are we in an Attic directory? Then we must check each
# filename for a "zombie" copy in the parent directory
# and decide which one to use.
in_attic = os.path.basename(dirpath) == 'Attic'
if in_attic:
parent = os.path.dirname(dirpath)
for filename in filenames:
# Ignore all non-RCS files.
if not filename.endswith(',v'):
continue
count += 1
progress(_('Collecting RCS files'), count)
#
# Perform the zombie check:
#
# 1.) If the zombie is in the parent directory, remove
# it from 'result' and add the one in the Attic.
#
# 2.) If the zombie is in the Attic, leave the good one
# in 'result' and ignore the one in the Attic.
#
# 3.) If neither of the two files can be classified as
# a zombie, raise an error.
#
if in_attic and self._zombie_check(result, parent, filename):
# This is case 2.) above: skip the Attic filename.
continue
filename = os.path.join(dirpath, filename)
if not self._unmodified(filename):
result.append(filename)
return result
def generate_changesets(self, progress=None, limit=None, flush=False):
"""Convert changes stored in the meta database into sets of
related changes and store the resulting changesets in the meta
database as well.
If limit is given, only that many changesets will be imported and
all others will be retained. This can be used to import changesets
in small batches.
If flush is True, all changesets produced by the ChangeSetGenerator
are imported; otherwise, potentially incomplete changesets may be
retained for the next incremental import. Use this flag if you can
be sure that the CVS repository is consistent and is not going to be
modified during the import.
"""
if progress == None:
progress = NoProgress()
with progress:
count = 0
csg = ChangeSetGenerator(limit=limit)
total = self.metadb.count_changes()
progress(_('Processing changes'), 0, total)
for change in self.changes(processed=False, reentrant=True):
count += 1
progress(_('Processing changes'), count, total)
for cs in csg.integrate(change):
self.metadb.add_changeset(cs)
if flush:
# All changesets are assumed to be complete and will be
# imported. Note that the ChangeSetGenerator still counts
# changesets from flush() against the specified limit.
for cs in csg.flush():
self.metadb.add_changeset(cs)
elif len(csg.changesets) > 0:
# The ChangeSetGenerator retained some changesets because
# they are potentially incomplete.
progress(_('Retained changesets'), len(csg.changesets))
def __init__(self, dirname, metadb):
self.metadb = metadb
# 'dirname' is a local filesystem path pointing at the root of
# a CVS repository or at a module within. If it is a module
# path, operations will be limited to that module. Otherwise,
# the module path will be empty and operations will by default
# apply to the whole repository.
#
# In any case, the repository path is always determined by
# recursively following the parents of 'dirname' until a a
# sub-directory named 'CVSROOT' is found. Repository path and
# module path are available for reading in the instance
# attributes 'root' and 'module', respectively.
if not os.path.isdir(dirname):
raise TypeError, _('not a CVS repository path (%s): %s') \
% (_('not even a directory'), dirname)
# Convert to absolute pathname, so that our logic doesn't
# break if anyone uses os.chdir() and so that the path
# traversal below can always assume an absolute path.
dirname = os.path.abspath(dirname)
# Split 'dirname' into self.root and self.module and also
# set self.prefix to the full absolute module path.
self.root, self.module = split_cvs_source(dirname)
if self.module == '':
self.prefix = self.root
else:
self.prefix = os.path.join(self.root, self.module)
self.localid = None
self.parse_config()
self.statcache = {}
self._rcs_log_keyword_re = re.compile('(.*)\$Log(?::[^$\r\n]+)?\$(.*)')
self._rcs_keyword_re = re.compile('\$([A-Z][A-Za-z]+)(:[^$\r\n]*)?\$')
self._rcs_strip_attic_re = re.compile('(Attic/)?([^/]+),v$')
def _main(self, argv):
self.option_parser = OptionParser(
prog=os.path.basename(argv[0]),
description=self.description,
usage=self.usage,
add_help_option=False)
# Turn off GNU-style mix of arguments and options. We have
# subcommands that accept --help, so the main command should
# not interpret --help after a subcommand name.
self.option_parser.allow_interspersed_args = False
self.add_option('--help', action='help', help=\
_("Show this help and exit."))
self.initialize_options()
self.options, self.args = self.option_parser.parse_args(argv[1:])
self.finalize_options()
sys.exit(self.run())
def run(self):
if os.path.exists(self.directory):
self.fatal(_("destination path '%s' already exists") % \
self.directory)
conduit = Conduit(self.directory)
conduit.init(self.repository,
bare=self.options.bare,
domain=self.options.domain,
quiet=self.options.quiet)
try:
conduit.fetch(limit=self.options.limit,
quiet=self.options.quiet,
verbose=self.options.verbose,
flush=self.options.no_skip_latest,
authors=self.options.authors,
stop_on_unknown_author=\
self.options.stop_on_unknown_author)
git = conduit.git
if not self.options.no_repack:
git.check_command('repack', '-adF')
head_branch = git.symbolic_ref('HEAD')
if head_branch == 'refs/heads/master':
if self.options.bare:
git.check_command('branch', '-f', 'master', conduit.branch)
else:
git.check_command('reset', '-q', '--hard', conduit.branch)
except:
shutil.rmtree(self.directory)
raise
# Verify after the above rmtree, because someone likely wants
# to inspect the repository if the verification fails.
if self.options.verify:
try:
olddir = os.getcwd()
os.chdir(git.git_work_tree)
Verify().eval()
finally:
os.chdir(olddir)
def __init__(self):
super(NoSourceError, self).__init__(
_("'cvs.source' is unset; not a git-cvs repository?"))
def add_quiet_option(self):
self.add_option('--quiet', action='store_true', help=\
_("Only report error and warning messages."))
def add_verbose_option(self):
self.add_option('--verbose', action='store_true', help=\
_("Display each changeset as it is imported."))
def add_stop_on_unknown_author_option(self):
self.add_option('--stop-on-unknown-author', action='store_true', help=\
_("Abort the operation if any author mapping is missing."))
def add_authors_option(self):
self.add_option('--authors', metavar='AUTHORS', help=\
_("Map CVS committer login names to fullnames."))
def fatal(self, msg):
sys.stderr.write(_('%s: fatal: %s\n') % (self.option_parser.prog, msg))
sys.exit(1)
def finalize_options(self):
if len(self.args) > 0:
self.usage_error(_('too many arguments'))
self.finalize_authors_option()
def initialize_options(self):
self.add_option('--checkout', metavar='REVISION', help=\
_("Dump the content of the specified REVISION."))
