def call_command(command, **kw): """Call the specified command, checking that it exits successfully. Raise a FatalError if the command cannot be executed, or if it exits with a non-zero exit code. Pass KW as keyword arguments to subprocess.call().""" logger.debug('Running command %r' % (command,)) try: retcode = subprocess.call(command, **kw) if retcode < 0: raise FatalError( 'Command terminated by signal %d: "%s"' % (-retcode, ' '.join(command),) ) elif retcode > 0: raise FatalError( 'Command failed with return code %d: "%s"' % (retcode, ' '.join(command),) ) except OSError, e: raise FatalError( 'Command execution failed (%s): "%s"' % (e, ' '.join(command),) )
def check(self): if Ctx().cross_project_commits: raise FatalError( 'Git output is not supported with cross-project commits') if Ctx().cross_branch_commits: raise FatalError( 'Git output is not supported with cross-branch commits') if Ctx().username is None: raise FatalError('Git output requires a default commit username')
def __init__( self, id, project_cvs_repos_path, initial_directories=[], symbol_transforms=None, exclude_paths=[], ): """Create a new Project record. ID is a unique id for this project. PROJECT_CVS_REPOS_PATH is the main CVS directory for this project (within the filesystem). INITIAL_DIRECTORIES is an iterable of all SVN directories that should be created when the project is first created. Normally, this should include the trunk, branches, and tags directory. SYMBOL_TRANSFORMS is an iterable of SymbolTransform instances which will be used to transform any symbol names within this project. EXCLUDE_PATHS is an iterable of paths that should be excluded from the conversion. The paths should be relative to PROJECT_CVS_REPOS_PATH and use slashes ('/'). Paths for individual files should include the ',v' extension. """ self.id = id self.project_cvs_repos_path = os.path.normpath(project_cvs_repos_path) if not os.path.isdir(self.project_cvs_repos_path): raise FatalError( "The specified CVS repository path '%s' is not an " "existing directory." % self.project_cvs_repos_path) self.cvs_repository_root, self.cvs_module = \ self.determine_repository_root( os.path.abspath(self.project_cvs_repos_path)) # The SVN directories to add when the project is first created: self._initial_directories = [] for path in initial_directories: try: path = normalize_svn_path(path, False) except IllegalSVNPathError, e: raise FatalError( 'Initial directory %r is not a legal SVN path: %s' % ( path, e, )) self._initial_directories.append(path)
def __init__( self, id, project_cvs_repos_path, initial_directories=[], symbol_transforms=None, ): """Create a new Project record. ID is a unique id for this project. PROJECT_CVS_REPOS_PATH is the main CVS directory for this project (within the filesystem). INITIAL_DIRECTORIES is an iterable of all SVN directories that should be created when the project is first created. Normally, this should include the trunk, branches, and tags directory. SYMBOL_TRANSFORMS is an iterable of SymbolTransform instances which will be used to transform any symbol names within this project.""" self.id = id self.project_cvs_repos_path = os.path.normpath(project_cvs_repos_path) if not os.path.isdir(self.project_cvs_repos_path): raise FatalError( "The specified CVS repository path '%s' is not an " "existing directory." % self.project_cvs_repos_path) self.cvs_repository_root, self.cvs_module = \ self.determine_repository_root( os.path.abspath(self.project_cvs_repos_path)) # A regexp matching project_cvs_repos_path plus an optional separator: self.project_prefix_re = re.compile( r'^' + re.escape(self.project_cvs_repos_path) + r'(' + re.escape(os.sep) + r'|$)') # The SVN directories to add when the project is first created: self._initial_directories = [] for path in initial_directories: try: path = normalize_svn_path(path, False) except IllegalSVNPathError, e: raise FatalError( 'Initial directory %r is not a legal SVN path: %s' % ( path, e, )) self._initial_directories.append(path)
def main(progname, run_options, pass_manager): # Convenience var, so we don't have to keep instantiating this Borg. ctx = Ctx() # Make sure the tmp directory exists. Note that we don't check if # it's empty -- we want to be able to use, for example, "." to hold # tempfiles. if ctx.tmpdir is None: ctx.tmpdir = tempfile.mkdtemp(prefix=('%s-' % (progname, ))) erase_tmpdir = True logger.quiet( 'Writing temporary files to %r\n' 'Be sure to use --tmpdir=%r if you need to resume this conversion.' % ( ctx.tmpdir, ctx.tmpdir, ), ) elif not os.path.exists(ctx.tmpdir): os.mkdir(ctx.tmpdir) erase_tmpdir = True elif not os.path.isdir(ctx.tmpdir): raise FatalError( "cvs2svn tried to use '%s' for temporary files, but that path\n" " exists and is not a directory. Please make it be a directory,\n" " or specify some other directory for temporary files." % (ctx.tmpdir, )) else: erase_tmpdir = False # But do lock the tmpdir, to avoid process clash. try: os.mkdir(os.path.join(ctx.tmpdir, 'cvs2svn.lock')) except OSError, e: if e.errno == errno.EACCES: raise FatalError("Permission denied:" + " No write access to directory '%s'." % ctx.tmpdir) if e.errno == errno.EEXIST: raise FatalError( "cvs2svn is using directory '%s' for temporary files, but\n" " subdirectory '%s/cvs2svn.lock' exists, indicating that another\n" " cvs2svn process is currently using '%s' as its temporary\n" " workspace. If you are certain that is not the case,\n" " then remove the '%s/cvs2svn.lock' subdirectory." % ( ctx.tmpdir, ctx.tmpdir, ctx.tmpdir, ctx.tmpdir, )) raise
def main(progname, cmd_args): # Disable garbage collection, as we try not to create any circular # data structures: gc.disable() # Convenience var, so we don't have to keep instantiating this Borg. ctx = Ctx() pass_manager = PassManager(passes) run_options = RunOptions(progname, cmd_args, pass_manager) # Make sure the tmp directory exists. Note that we don't check if # it's empty -- we want to be able to use, for example, "." to hold # tempfiles. But if we *did* want check if it were empty, we'd do # something like os.stat(ctx.tmpdir)[stat.ST_NLINK], of course :-). if not os.path.exists(ctx.tmpdir): erase_tmpdir = True os.mkdir(ctx.tmpdir) elif not os.path.isdir(ctx.tmpdir): raise FatalError( "cvs2svn tried to use '%s' for temporary files, but that path\n" " exists and is not a directory. Please make it be a directory,\n" " or specify some other directory for temporary files." % (ctx.tmpdir, )) else: erase_tmpdir = False # But do lock the tmpdir, to avoid process clash. try: os.mkdir(os.path.join(ctx.tmpdir, 'cvs2svn.lock')) except OSError, e: if e.errno == errno.EACCES: raise FatalError("Permission denied:" + " No write access to directory '%s'." % ctx.tmpdir) if e.errno == errno.EEXIST: raise FatalError( "cvs2svn is using directory '%s' for temporary files, but\n" " subdirectory '%s/cvs2svn.lock' exists, indicating that another\n" " cvs2svn process is currently using '%s' as its temporary\n" " workspace. If you are certain that is not the case,\n" " then remove the '%s/cvs2svn.lock' subdirectory." % ( ctx.tmpdir, ctx.tmpdir, ctx.tmpdir, ctx.tmpdir, )) raise
def __init__(self, value): if value not in ['collapsed', 'expanded', 'untouched', None]: raise FatalError( 'Value for %s must be "collapsed", "expanded", or "untouched"' % (self.propname,) ) self.value = value
def _get_cvs_file( self, parent_directory, basename, file_in_attic=False, leave_in_attic=False, ): """Return a CVSFile describing the file with name BASENAME. PARENT_DIRECTORY is the CVSDirectory instance describing the directory that physically holds this file in the filesystem. BASENAME must be the base name of a *,v file within PARENT_DIRECTORY. FILE_IN_ATTIC is a boolean telling whether the specified file is in an Attic subdirectory. If FILE_IN_ATTIC is True, then: - If LEAVE_IN_ATTIC is True, then leave the 'Attic' component in the filename. - Otherwise, raise FileInAndOutOfAtticException if a file with the same filename appears outside of Attic. The CVSFile is assigned a new unique id. All of the CVSFile information is filled in except mode (which can only be determined by parsing the file). Raise FatalError if the resulting filename would not be legal in SVN.""" filename = os.path.join(parent_directory.rcs_path, basename) try: Ctx().output_option.verify_filename_legal(basename[:-2]) except IllegalSVNPathError, e: raise FatalError( 'File %r would result in an illegal SVN filename: %s' % (filename, e,) )
def process_io_options(self): """Process input/output options. Process options related to extracting data from the CVS repository and writing to a Bazaar-friendly fast-import file.""" ctx = Ctx() options = self.options not_both(options.use_rcs, '--use-rcs', options.use_cvs, '--use-cvs') if options.use_rcs: revision_reader = RCSRevisionReader( co_executable=options.co_executable) else: # --use-cvs is the default: revision_reader = CVSRevisionReader( cvs_executable=options.cvs_executable) if not ctx.dry_run and not options.dumpfile: raise FatalError("must pass '--dry-run' or '--dumpfile' option.") ctx.revision_recorder = NullRevisionRecorder() ctx.revision_excluder = NullRevisionExcluder() ctx.revision_reader = None ctx.output_option = GitOutputOption( options.dumpfile, GitRevisionInlineWriter(revision_reader), max_merges=None, # Optional map from CVS author names to bzr author names: author_transforms={}, # FIXME )
def callback_symbol_transform(self, option, opt_str, value, parser): [pattern, replacement] = value.split(":") try: parser.values.symbol_transforms.append( RegexpSymbolTransform(pattern, replacement)) except re.error: raise FatalError("'%s' is not a valid regexp." % (pattern, ))
def get_content(self, cvs_rev): """Check out the text for revision C_REV from the repository. Return the text. If CVS_REV has a property _keyword_handling, use it to determine how to handle RCS keywords in the output: 'collapsed' -- collapse keywords 'expanded' -- expand keywords 'untouched' -- output keywords in the form they are found in the RCS file Note that $Log$ never actually generates a log (which makes test 'requires_cvs()' fail). Revisions may be requested in any order, but if they are not requested in dependency order the checkout database will become very large. Revisions may be skipped. Each revision may be requested only once.""" try: text = self._get_text_record(cvs_rev).checkout( self._text_record_db) except MalformedDeltaException, (msg): raise FatalError('Malformed RCS delta in %s, revision %s: %s' % (cvs_rev.cvs_file.rcs_path, cvs_rev.rev, msg))
def __init__(self, cvs_executable, global_options=None): """Initialize a CVSRevisionReader. CVS_EXECUTABLE is the CVS command (possibly including the full path to the executable; otherwise it is sought in the $PATH). GLOBAL_ARGUMENTS, if specified, should be a list of global options that are passed to the CVS command before the subcommand. If GLOBAL_ARGUMENTS is not specified, then each of the possibilities listed in _possible_global_options is checked in order until one is found that runs successfully and without any output to stderr.""" self.cvs_executable = cvs_executable if global_options is None: for global_options in self._possible_global_options: try: self._check_cvs_runs(global_options) except CommandFailedException, e: pass else: break else: raise FatalError( '%s\n' 'Please check that cvs is installed and in your PATH.' % (e, ))
def determine_repository_root(path): """Ascend above the specified PATH if necessary to find the cvs_repository_root (a directory containing a CVSROOT directory) and the cvs_module (the path of the conversion root within the cvs repository). Return the root path and the module path of this project relative to the root. NB: cvs_module must be seperated by '/', *not* by os.sep.""" def is_cvs_repository_root(path): return os.path.isdir(os.path.join(path, 'CVSROOT')) original_path = path cvs_module = '' while not is_cvs_repository_root(path): # Step up one directory: prev_path = path path, module_component = os.path.split(path) if path == prev_path: # Hit the root (of the drive, on Windows) without finding a # CVSROOT dir. raise FatalError( "the path '%s' is not a CVS repository, nor a path " "within a CVS repository. A CVS repository contains " "a CVSROOT directory within its root directory." % (original_path, )) cvs_module = module_component + "/" + cvs_module return path, cvs_module
def get_content(self, cvs_rev): # Is EOL fixing requested? eol_fix = cvs_rev.get_property('_eol_fix') or None # How do we want keywords to be handled? keyword_handling = cvs_rev.get_property('_keyword_handling') or None try: (k_option, explicit_keyword_handling) = self._text_options[bool(eol_fix), keyword_handling] except KeyError: raise FatalError( 'Undefined _keyword_handling property (%r) for %s' % ( keyword_handling, cvs_rev, )) data = get_command_output(self.get_pipe_command(cvs_rev, k_option)) if Ctx().decode_apple_single: # Insert a filter to decode any files that are in AppleSingle # format: data = get_maybe_apple_single(data) if explicit_keyword_handling == 'expanded': data = expand_keywords(data, cvs_rev) elif explicit_keyword_handling == 'collapsed': data = collapse_keywords(data) if eol_fix: data = canonicalize_eol(data, eol_fix) return data
def process_output_options(self): """Process options related to fastimport output.""" ctx = Ctx() options = self.options if options.use_rcs: revision_reader = RCSRevisionReader( co_executable=options.co_executable) else: # --use-cvs is the default: revision_reader = CVSRevisionReader( cvs_executable=options.cvs_executable) if not ctx.dry_run and not options.dumpfile: raise FatalError("must pass '--dry-run' or '--dumpfile' option.") # See cvs2bzr-example.options for explanations of these ctx.revision_collector = NullRevisionCollector() ctx.revision_reader = None if ctx.dry_run: ctx.output_option = NullOutputOption() else: ctx.output_option = BzrOutputOption( options.dumpfile, GitRevisionInlineWriter(revision_reader), # Optional map from CVS author names to bzr author names: author_transforms={}, # FIXME )
def write(self, s): try: self.loader_pipe.stdin.write(s) except IOError: raise FatalError('svnadmin failed with the following output while ' 'loading the dumpfile:\n%s' % (self.loader_pipe.stderr.read(), ))
def compute_best_source(self, preferred_source): """Determine the best source_lod and subversion revision number to copy. Return the best source found, as an SVNRevisionRange instance. If PREFERRED_SOURCE is not None and its opening is among the sources with the best scores, return it; otherwise, return the oldest such revision on the first such source_lod (ordered by the natural LOD sort order). The return value's source_lod is the best LOD to copy from, and its opening_revnum is the best SVN revision.""" # Aggregate openings and closings from our rev tree svn_revision_ranges = self._get_revision_ranges(self._node_tree) # Score the lists revision_scores = RevisionScores(svn_revision_ranges) best_source_lod, best_revnum, best_score = \ revision_scores.get_best_revnum() if (preferred_source is not None and revision_scores.get_score(preferred_source) == best_score): best_source_lod = preferred_source.source_lod best_revnum = preferred_source.opening_revnum if best_revnum == SVN_INVALID_REVNUM: raise FatalError( "failed to find a revision to copy from when copying %s" % self._symbol.name) return SVNRevisionRange(best_source_lod, best_revnum)
def normalize_ttb_path(opt, path, allow_empty=False): try: return normalize_svn_path(path, allow_empty) except IllegalSVNPathError, e: raise FatalError('Problem with %s: %s' % ( opt, e, ))
def __init__(self, co_executable): self.co_executable = co_executable try: check_command_runs([self.co_executable, '-V'], self.co_executable) except CommandFailedException, e: raise FatalError('%s\n' 'Please check that co is installed and in your PATH\n' '(it is a part of the RCS software).' % (e,))
def transform(self, cvs_file, symbol_name, revision): try: return normalize_svn_path(symbol_name) except IllegalSVNPathError, e: raise FatalError('Problem with %s: %s' % ( symbol_name, e, ))
def process_output_options(self): """Process the options related to SVN output.""" ctx = Ctx() options = self.options if options.dump_only and not options.dumpfile: raise FatalError( "'--dump-only' requires '--dumpfile' to be specified.") if not options.svnrepos and not options.dumpfile and not ctx.dry_run: raise FatalError("must pass one of '-s' or '--dumpfile'.") not_both(options.svnrepos, '-s', options.dumpfile, '--dumpfile') not_both(options.dumpfile, '--dumpfile', options.existing_svnrepos, '--existing-svnrepos') not_both(options.bdb_txn_nosync, '--bdb-txn-nosync', options.existing_svnrepos, '--existing-svnrepos') not_both(options.dumpfile, '--dumpfile', options.bdb_txn_nosync, '--bdb-txn-nosync') not_both(options.fs_type, '--fs-type', options.existing_svnrepos, '--existing-svnrepos') if (options.fs_type and options.fs_type != 'bdb' and options.bdb_txn_nosync): raise FatalError( "cannot pass --bdb-txn-nosync with --fs-type=%s." % options.fs_type) if options.svnrepos: if options.existing_svnrepos: ctx.output_option = ExistingRepositoryOutputOption( options.svnrepos) else: ctx.output_option = NewRepositoryOutputOption( options.svnrepos, fs_type=options.fs_type, bdb_txn_nosync=options.bdb_txn_nosync, create_options=options.create_options) else: ctx.output_option = DumpfileOutputOption(options.dumpfile)
def finish(self): self._popen.stdin.close() logger.normal('Waiting for generate_blobs.py to finish...') returncode = self._popen.wait() if returncode: raise FatalError('generate_blobs.py failed with return code %s.' % (returncode, )) else: logger.normal('generate_blobs.py is done.')
def callback_encoding(self, option, opt_str, value, parser): ctx = Ctx() try: ctx.cvs_author_decoder.add_encoding(value) ctx.cvs_log_decoder.add_encoding(value) ctx.cvs_filename_decoder.add_encoding(value) except LookupError, e: raise FatalError(str(e))
def callback_fallback_encoding(self, option, opt_str, value, parser): ctx = Ctx() try: ctx.cvs_author_decoder.set_fallback_encoding(value) ctx.cvs_log_decoder.set_fallback_encoding(value) # Don't use fallback_encoding for filenames. except LookupError, e: raise FatalError(str(e))
def utf8_path(path): """Return a copy of PATH encoded in UTF-8.""" try: return Ctx().cvs_filename_decoder.decode_path(path).encode('utf8') except UnicodeError: raise FatalError( "Unable to convert a path '%s' to internal encoding.\n" "Consider rerunning with one or more '--encoding' parameters or\n" "with '--fallback-encoding'." % (path, ))
def check_options(self): """Check the the run options are OK. This should only be called after all options have been processed.""" # Convenience var, so we don't have to keep instantiating this Borg. ctx = Ctx() if not self.start_pass <= self.end_pass: raise InvalidPassError( 'Ending pass must not come before starting pass.') if not ctx.dry_run and ctx.output_option is None: raise FatalError('No output option specified.') if ctx.output_option is not None: ctx.output_option.check() if not self.projects: raise FatalError('No project specified.')
def get_symbol(self, symbol, stats): if not isinstance(symbol, LineOfDevelopment): # This symbol will not be included in the conversion; skip it. return symbol if symbol.base_path is not None: # This lod's base path is already set; leave it. pass elif isinstance(symbol, Trunk): trunk_path = symbol.project.trunk_path if trunk_path is None: raise FatalError('DefaultBasePathRule used for trunk,\n' 'but project\'s trunk path is not set') symbol.base_path = trunk_path elif isinstance(symbol, Branch): branches_path = symbol.project.branches_path if branches_path is None: raise FatalError('DefaultBasePathRule used for symbol %s,\n' 'but project\'s branch path is not set' % (symbol, )) symbol.base_path = path_join(branches_path, symbol.name) elif isinstance(symbol, Tag): tags_path = symbol.project.tags_path if tags_path is None: raise FatalError('DefaultBasePathRule used for symbol %s,\n' 'but project\'s tag path is not set' % (symbol, )) symbol.base_path = path_join(tags_path, symbol.name) else: raise NotImplementedError() return symbol
def _utf8_path(self, path): """Return a copy of PATH encoded in UTF-8.""" # Convert each path component separately (as they may each use # different encodings). try: return '/'.join([ Ctx().cvs_filename_decoder(piece).encode('utf8') for piece in path.split('/') ]) except UnicodeError: raise FatalError( "Unable to convert a path '%s' to internal encoding.\n" "Consider rerunning with one or more '--encoding' parameters or\n" "with '--fallback-encoding'." % (path, ))
def _mutate_branch_to_tag(self, cvs_branch): """Mutate the branch CVS_BRANCH into a tag.""" if cvs_branch.next_id is not None: # This shouldn't happen because it was checked in # CollateSymbolsPass: raise FatalError('Attempt to exclude a branch with commits.') cvs_tag = CVSTag( cvs_branch.id, cvs_branch.cvs_file, cvs_branch.symbol, cvs_branch.source_lod, cvs_branch.source_id, cvs_branch.revision_reader_token, ) self.add(cvs_tag) cvs_revision = self[cvs_tag.source_id] cvs_revision.branch_ids.remove(cvs_tag.id) cvs_revision.tag_ids.append(cvs_tag.id)
def end_commit(self): """Feed the revision stored in the dumpfile to the svnadmin load pipe.""" DumpfileDelegate.end_commit(self) self.dumpfile.seek(0) while True: data = self.dumpfile.read(128 * 1024) # Chunk size is arbitrary if not data: break try: self.loader_pipe.stdin.write(data) except IOError: raise FatalError("svnadmin failed with the following output " "while loading the dumpfile:\n" + self.loader_pipe.stderr.read()) self.dumpfile.seek(0) self.dumpfile.truncate()
def __init__(self, msg): FatalError.__init__( self, msg + '\nUse --help-passes for more information.')