def test_checks_uuid(self): repos_url = self.make_svn_repository('d') dc = self.get_commit_editor(repos_url) dc.add_dir("bp") dc.close() mapping = mapping_registry.get_default()() ra = RemoteAccess(repos_url.encode("utf-8"), auth=Auth([get_username_provider()])) revnum = ra.get_latest_revnum() revprops = { SVN_REVPROP_BZR_REPOS_UUID: "otheruuid", "svn:log": "bla", SVN_REVPROP_BZR_ROOT: "bp", SVN_REVPROP_BZR_MAPPING_VERSION: mapping.name, SVN_REVPROP_BZR_BASE_REVISION: "therealbaserevid" } dc = TestCommitEditor(ra.get_commit_editor(revprops), ra.url, revnum) dc.open_dir("bp").add_file("bp/la").modify() dc.close() repos = Repository.open(repos_url) revmeta1 = repos._revmeta_provider.get_revision(u"bp", 1) revmeta2 = repos._revmeta_provider.get_revision(u"bp", 2) self.assertEquals( mapping.revision_id_foreign_to_bzr((repos.uuid, "bp", 1)), revmeta2.get_lhs_parent_revid(mapping, revmeta1))
def get_commit_editor(self, url, message="Test commit"): """Obtain a commit editor. :param url: URL to connect to :param message: Commit message :return: Commit editor object """ ra_ctx = RemoteAccess(url.encode("utf-8"), auth=Auth([ra.get_username_provider()])) revnum = ra_ctx.get_latest_revnum() return TestCommitEditor(ra_ctx.get_commit_editor({"svn:log": message}), ra_ctx.url, revnum)
def find_root(self, uuid, url): for root in self._roots[uuid]: if url.startswith(root): return root try: from subvertpy.ra import RemoteAccess except ImportError: return None c = RemoteAccess(url) root = c.get_repos_root() self._roots[uuid].add(root) return root
def test_remote_access_retry_failure( mocker, tmp_path, sample_repo_url, exception_to_retry ): nb_failed_calls = SVN_RETRY_MAX_ATTEMPTS mock_ra = mocker.patch("swh.loader.svn.svn.RemoteAccess") remote_access = RemoteAccess(sample_repo_url, auth=Auth([get_username_provider()])) mock_ra.side_effect = ( [exception_to_retry] * nb_failed_calls + [remote_access] + [exception_to_retry] * nb_failed_calls + [remote_access] ) mock_sleep = mocker.patch.object(SvnRepo.remote_access.retry, "sleep") with pytest.raises(type(exception_to_retry)): SvnRepo( sample_repo_url, sample_repo_url, tmp_path, max_content_length=100000, ) assert_sleep_calls(mock_sleep, mocker, nb_failed_calls - 1)
def Connection(url, auth=None, config=None, readonly=False): progress_cb = SubversionProgressReporter(url).update try: ret = RemoteAccess( _url_escape_uri(url), auth=auth, client_string_func=breezy.plugins.svn.get_client_string, progress_cb=progress_cb, config=config) if 'transport' in debug.debug_flags: ret = MutteringRemoteAccess(ret) if readonly: ret = ReadonlyRemoteAccess(ret) except subvertpy.SubversionException as e: msg, num = e.args if num in (subvertpy.ERR_RA_SVN_REPOS_NOT_FOUND, ): raise NoSvnRepositoryPresent(url=url) if num == subvertpy.ERR_BAD_URL: raise urlutils.InvalidURL(url) if num in (subvertpy.ERR_RA_DAV_PATH_NOT_FOUND, subvertpy.ERR_FS_NOT_FOUND): raise NoSuchFile(url) if num == subvertpy.ERR_RA_ILLEGAL_URL: raise urlutils.InvalidURL(url, msg) if num == subvertpy.ERR_RA_DAV_RELOCATED: raise convert_relocate_error(url, num, msg) raise convert_error(e) from . import lazy_check_versions lazy_check_versions() return ret
def __init__(self, url, output, rev_map={}, author_map=None, root=None, ignore=(), export_copies=False, quiet=False, ): self.output = output self.author_map = author_map self.ignore = ignore self.export_copies = export_copies self.known_branches = defaultdict(lambda: (list(), list())) for (branch, revs) in rev_map.items(): branch = branch.lstrip("/") (starts, runs) = self.known_branches[branch] start = None run = () for (svnrev, gitrev) in sorted(revs.items()): if svnrev - len(run) != start: start = svnrev starts.append(start) run = list() runs.append(run) run.append(gitrev) self.quiet = quiet if self.quiet: self.progress = dummycontext else: self.progress = progresscontext auth = subvertpy.ra.Auth(( # Avoids the following error for diffs on local (file:) URLs: # "No provider registered for 'svn.username' credentials" subvertpy.ra.get_username_provider(), # Avoids RemoteAccess() failing for HTTPS URLs with # "Unable to connect to a repository at URL" # and error code 215001 ("No authentication provider available") subvertpy.ra.get_ssl_server_trust_file_provider(), )) with self.progress("connecting to ", url): self.ra = RemoteAccess(url, auth=auth) self.url = url self.repos_root = self.ra.get_repos_root() if root is None: self.root = self.repos_root else: self.root = root self.uuid = self.ra.get_uuid()
def add_commit(repo_url: str, message: str, changes: List[CommitChange]) -> None: conn = RemoteAccess(repo_url, auth=Auth([get_username_provider()])) editor = conn.get_commit_editor({"svn:log": message}) root = editor.open_root() for change in changes: if change["change_type"] == CommitChangeType.Delete: root.delete_entry(change["path"].rstrip("/")) else: dir_change = change["path"].endswith("/") split_path = change["path"].rstrip("/").split("/") for i in range(len(split_path)): path = "/".join(split_path[0 : i + 1]) if i < len(split_path) - 1: try: root.add_directory(path).close() except SubversionException: pass else: if dir_change: try: dir = root.add_directory(path) except SubversionException: dir = root.open_directory(path) if "properties" in change: for prop, value in change["properties"].items(): dir.change_prop(prop, value) dir.close() else: try: file = root.add_file(path) except SubversionException: file = root.open_file(path) if "properties" in change: for prop, value in change["properties"].items(): file.change_prop(prop, value) if "data" in change: txdelta = file.apply_textdelta() delta.send_stream(BytesIO(change["data"]), txdelta) file.close() root.close() editor.close()
def main(local_url, svn_url, revision_start, revision_end, debug, cleanup): """Script to present how to use Replay class.""" conn = RemoteAccess(svn_url.encode("utf-8"), auth=Auth([get_username_provider()])) os.makedirs(local_url, exist_ok=True) rootpath = tempfile.mkdtemp( prefix=local_url, suffix="-" + os.path.basename(svn_url) ) rootpath = os.fsencode(rootpath) # Do not go beyond the repository's latest revision revision_end_max = conn.get_latest_revnum() if revision_end == -1: revision_end = revision_end_max revision_end = min(revision_end, revision_end_max) try: replay = Replay(conn, rootpath) for rev in range(revision_start, revision_end + 1): contents, skipped_contents, directories = replay.compute_objects(rev) print( "r%s %s (%s new contents, %s new directories)" % ( rev, hashutil.hash_to_hex(replay.directory.hash), len(contents) + len(skipped_contents), len(directories), ) ) if debug: print("%s" % rootpath.decode("utf-8")) finally: if cleanup: if os.path.exists(rootpath): shutil.rmtree(rootpath)
def test_remote_access_retry_success( mocker, tmp_path, sample_repo_url, exception_to_retry ): nb_failed_calls = 2 mock_ra = mocker.patch("swh.loader.svn.svn.RemoteAccess") remote_access = RemoteAccess(sample_repo_url, auth=Auth([get_username_provider()])) mock_ra.side_effect = ( [exception_to_retry] * nb_failed_calls + [remote_access] + [exception_to_retry] * nb_failed_calls + [remote_access] ) mock_sleep = mocker.patch.object(SvnRepo.remote_access.retry, "sleep") SvnRepo( sample_repo_url, sample_repo_url, tmp_path, max_content_length=100000, ) assert_sleep_calls(mock_sleep, mocker, nb_failed_calls)
class RecordingRemoteAccess(object): """Trivial RemoteAccess wrapper that records all activity.""" busy = property(lambda self: self.actual.busy) url = property(lambda self: self.actual.url) def __init__(self, *args, **kwargs): self.actual = RemoteAccess(*args, **kwargs) def check_path(self, path, revnum): self.__class__.calls.append(("check-path", (revnum, path))) return self.actual.check_path(path, revnum) def stat(self, path, revnum): self.__class__.calls.append(("stat", (revnum, path))) return self.actual.stat(path, revnum) def has_capability(self, cap): self.__class__.calls.append(("has-capability", (cap, ))) return self.actual.has_capability(cap) def get_uuid(self): self.__class__.calls.append(("get-uuid", ())) return self.actual.get_uuid() def get_repos_root(self): self.__class__.calls.append(("get-repos", ())) return self.actual.get_repos_root() def get_latest_revnum(self): self.__class__.calls.append(("get-latest-revnum", ())) return self.actual.get_latest_revnum() def get_log(self, callback, paths, from_revnum, to_revnum, limit, *args, **kwargs): self.__class__.calls.append( ("log", (from_revnum, to_revnum, paths, limit))) return self.actual.get_log(callback, paths, from_revnum, to_revnum, limit, *args, **kwargs) def iter_log(self, paths, from_revnum, to_revnum, limit, *args, **kwargs): self.__class__.calls.append( ("log", (from_revnum, to_revnum, paths, limit))) return self.actual.iter_log(paths, from_revnum, to_revnum, limit, *args, **kwargs) def change_rev_prop(self, revnum, name, value): self.__class__.calls.append(("change-revprop", (revnum, name, value))) return self.actual.change_rev_prop(revnum, name, value) def get_dir(self, path, revnum=-1, fields=0): self.__class__.calls.append(("get-dir", (path, revnum, fields))) return self.actual.get_dir(path, revnum, fields) def get_file(self, path, stream, revnum): self.__class__.calls.append(("get-file", (path, revnum))) return self.actual.get_file(path, stream, revnum) def get_file_revs(self, path, start_revnum, end_revnum, handler): self.__class__.calls.append( ("get-file-revs", (path, start_revnum, end_revnum))) return self.actual.get_file_revs(path, start_revnum, end_revnum, handler) def revprop_list(self, revnum): self.__class__.calls.append(("revprop-list", (revnum, ))) return self.actual.revprop_list(revnum) def get_locations(self, path, peg_revnum, revnums): self.__class__.calls.append( ("get-locations", (path, peg_revnum, revnums))) return self.actual.get_locations(path, peg_revnum, revnums) def do_update(self, revnum, path, start_empty, editor): self.__class__.calls.append(("do-update", (revnum, path, start_empty))) return self.actual.do_update(revnum, path, start_empty, editor) def do_diff(self, revision_to_update, diff_target, versus_url, diff_editor, recurse=True, ignore_ancestry=False, text_deltas=False, depth=None): self.__class__.calls.append(("diff", (revision_to_update, diff_target, versus_url, text_deltas, depth))) return self.actual.do_diff(revision_to_update, diff_target, versus_url, diff_editor, recurse, ignore_ancestry, text_deltas) def do_switch(self, revnum, path, start_empty, to_url, editor): self.__class__.calls.append( ("switch", (revnum, path, start_empty, to_url))) return self.actual.do_switch(revnum, path, start_empty, to_url, editor) def reparent(self, url): self.__class__.calls.append(("reparent", (url, ))) return self.actual.reparent(url) def get_commit_editor(self, *args, **kwargs): self.__class__.calls.append(("commit", ())) return self.actual.get_commit_editor(*args, **kwargs) def rev_proplist(self, revnum): self.__class__.calls.append(("rev-proplist", (revnum, ))) return self.actual.rev_proplist(revnum) def replay_range(self, start_revision, end_revision, low_water_mark, cbs, send_deltas=True): self.__class__.calls.append( ("replay-range", (start_revision, end_revision, low_water_mark, send_deltas))) return self.actual.replay_range(start_revision, end_revision, low_water_mark, cbs, send_deltas) def replay(self, revision, low_water_mark, editor, send_deltas=True): self.__class__.calls.append( ("replay", (revision, low_water_mark, send_deltas))) return self.actual.replay(revision, low_water_mark, editor, send_deltas)
class Exporter: def __init__(self, url, output, rev_map={}, author_map=None, root=None, ignore=(), export_copies=False, quiet=False, ): self.output = output self.author_map = author_map self.ignore = ignore self.export_copies = export_copies self.known_branches = defaultdict(lambda: (list(), list())) for (branch, revs) in rev_map.items(): branch = branch.lstrip("/") (starts, runs) = self.known_branches[branch] start = None run = () for (svnrev, gitrev) in sorted(revs.items()): if svnrev - len(run) != start: start = svnrev starts.append(start) run = list() runs.append(run) run.append(gitrev) self.quiet = quiet if self.quiet: self.progress = dummycontext else: self.progress = progresscontext auth = subvertpy.ra.Auth(( # Avoids the following error for diffs on local (file:) URLs: # "No provider registered for 'svn.username' credentials" subvertpy.ra.get_username_provider(), # Avoids RemoteAccess() failing for HTTPS URLs with # "Unable to connect to a repository at URL" # and error code 215001 ("No authentication provider available") subvertpy.ra.get_ssl_server_trust_file_provider(), )) with self.progress("connecting to ", url): self.ra = RemoteAccess(url, auth=auth) self.url = url self.repos_root = self.ra.get_repos_root() if root is None: self.root = self.repos_root else: self.root = root self.uuid = self.ra.get_uuid() def export(self, git_ref, branch=None, rev=INVALID_REVNUM): if branch is None: branch = self.url[len(self.repos_root) + 1:] self.git_ref = git_ref segments = PendingSegments(self, branch, rev) # Not using RemoteAccess.get_file_revs() because it does not work on # directories # TODO: Use RemoteAccess.replay_range() for initial location segment # and trailing parts of subsequent segments. Would require # remembering all versions of files received. (base_rev, base_path) = segments.base if base_rev: gitrev = segments.git_base else: gitrev = None init_export = True for (base, end, path) in segments: path = "/" + path prefix = path.rstrip("/") + "/" url = (self.repos_root + path).rstrip("/") with iter_revs(self, path, base, end) as revs: for (svnrev, date, author, log, self.paths) in revs: commit = self.export_copies # Assuming we are only interested in "trunk": # A /proj2/trunk from /proj1/trunk -> no commit # A /proj2 from /proj1 -> no commit # A /trunk without copy -> commit # A /proj/trunk from /proj/branch -> no commit commit = commit or any(path.startswith(prefix) and path > prefix for path in self.paths.keys()) if not commit: default = (None, None, None) (_, src, _) = self.paths.get(path, default) commit = src is None if commit: gitrev = self.commit(svnrev, date, author, log, init_export=init_export, base_rev=base_rev, base_path=base_path, gitrev=gitrev, path=path, prefix=prefix, url=url, ) init_export = False else: self.log(": no changes") self.output.printf("reset {}", git_ref) self.output.printf("from {}", gitrev) base_rev = svnrev base_path = path[1:] # Remember newly exported Git revision (svnstarts, gitruns) = self.known_branches[base_path] i = bisect_left(svnstarts, base_rev) if (i > 0 and svnstarts[i - 1] + len(gitruns[i - 1]) == base_rev): gitruns[i - 1].append(gitrev) else: svnstarts.insert(i, base_rev) gitruns.insert(i, [gitrev]) return gitrev def commit(self, rev, date, author, log, *, init_export, base_rev, base_path, gitrev, path, prefix, url): if not init_export and base_path != path[1:]: # Base revision is at a different branch location. # Will have to diff the base location against the # current location. Have to switch root because the # diff reporter does not accept link_path() on the # top-level directory. self.url = self.repos_root + "/" + base_path self.url = self.url.rstrip("/") self.ra.reparent(self.url) self.log(":") editor = RevEditor(self.output, self.quiet) # Diff editor does not convey deletions when starting # from scratch if init_export: dir = DirEditor(editor) for (file, (action, _, _)) in self.paths.items(): if not file.startswith(prefix) or action not in "DR": continue file = file[len(prefix):] for p in self.ignore: if file == p or file.startswith((p + "/").lstrip("/")): break else: dir.delete_entry(file) reporter = self.ra.do_diff(rev, "", url, editor, True, True, True) if init_export: reporter.set_path("", rev, True) else: reporter.set_path("", base_rev, False) for p in self.ignore: reporter.set_path(p, INVALID_REVNUM, True, None, subvertpy.ra.DEPTH_EXCLUDE) reporter.finish() # Assume the editor calls are all completed now merges = list() if editor.mergeinfo: self.log("\n") basehist = Ancestors(self) if base_rev: basehist.add_natural(base_path, base_rev) merged = RevisionSet() ancestors = Ancestors(self) merged.update(basehist) mergeinfo = editor.mergeinfo.items() for (branch, ranges) in mergeinfo: for (start, end, _) in ranges: merged.add_segment(branch, start, end) ancestors.add_natural(branch, end) if merged != basehist and ancestors == merged: # TODO: minimise so that only independent branch heads are listed # i.e. do not explicitly merge C if also merging A and B, and C is an ancestor of both A and B for (branch, ranges) in mergeinfo: branch = branch.lstrip("/") for (_, end, _) in ranges: ancestor = self.export(self.git_ref, branch, end) if ancestor is not None: merges.append(ancestor) self.output.printf("commit {}", self.git_ref) mark = self.output.newmark() self.output.printf("mark {}", mark) date = time_from_cstring(date) // 10**6 if self.author_map is None: author = "{author} <{author}@{uuid}>".format( author=author, uuid=self.uuid) else: author = self.author_map[author] self.output.printf("committer {} {} +0000", author, date) log = "{}\n\ngit-svn-id: {}{}@{} {}\n".format( log, self.root, path.rstrip("/"), rev, self.uuid) log = log.encode("utf-8") self.output.printf("data {}", len(log)) self.output.file.write(log) self.output.printf("") if (init_export or merges) and gitrev is not None: self.output.printf("from {}", gitrev) for ancestor in merges: self.output.printf("merge {}", ancestor) for line in editor.edits: self.output.printf("{}", line) self.output.printf("") return mark def log(self, message): if not self.quiet: stderr.write(message) stderr.flush()
#!/usr/bin/python # Demonstrates how to iterate over the log of a Subversion repository. from subvertpy.ra import RemoteAccess conn = RemoteAccess("svn://svn.samba.org/subvertpy/trunk") for (changed_paths, rev, revprops, has_children) in conn.iter_log( paths=None, start=0, end=conn.get_latest_revnum(), discover_changed_paths=True ): print "=" * 79 print "%d:" % rev print "Revision properties:" for entry in revprops.items(): print " %s: %s" % entry print "" print "Changed paths" for path, (action, from_path, from_rev, node_kind) in changed_paths.iteritems(): print " %s (%s)" % (path, action)
#!/usr/bin/python import cmd import subvertpy from subvertpy.ra import RemoteAccess import sys if len(sys.argv) == 1: print("Usage: %s <url>" % sys.argv) url = sys.argv[1] conn = RemoteAccess(url) def log_printer(changed_paths, rev, revprops, has_children=None): print("=" * 79) print("%d:" % rev) print("Revision properties:") for entry in revprops.items(): print(" %s: %s" % entry) print("") if changed_paths is None: return print("Changed paths:") for path, (action, from_path, from_rev) in changed_paths.items(): print(" %s (%s)" % (path, action)) class RaCmd(cmd.Cmd):
#!/usr/bin/python # Demonstrates how to iterate over the log of a Subversion repository. from subvertpy.ra import RemoteAccess conn = RemoteAccess("svn://svn.samba.org/subvertpy/trunk") for (changed_paths, rev, revprops, has_children) in conn.iter_log(paths=None, start=0, end=conn.get_latest_revnum(), discover_changed_paths=True): print("=" * 79) print("%d:" % rev) print("Revision properties:") for entry in revprops.items(): print(" %s: %s" % entry) print("") print("Changed paths") for path, (action, from_path, from_rev, node_kind) in (changed_paths.items()): print(" %s (%s)" % (path, action))
#!/usr/bin/python # Demonstrates how to do a new commit using Subvertpy import os from cStringIO import StringIO from subvertpy import delta, repos from subvertpy.ra import RemoteAccess, Auth, get_username_provider # Create a repository repos.create("tmprepo") # Connect to the "remote" repository using the file transport. # Note that a username provider needs to be provided, so that Subversion # knows who to record as the author of new commits made over this connection. repo_url = "file://%s" % os.path.abspath("tmprepo") conn = RemoteAccess(repo_url, auth=Auth([get_username_provider()])) # Simple commit that adds a directory editor = conn.get_commit_editor({"svn:log": "Commit message"}) root = editor.open_root() # Add a directory dir = root.add_directory("somedir") dir.close() # Add and edit a file file = root.add_file("somefile") # Set the svn:executable attribute file.change_prop("svn:executable", "*") # Obtain a textdelta handler and send the new file contents txdelta = file.apply_textdelta() delta.send_stream(StringIO("new file contents"), txdelta) file.close()
def __init__(self, *args, **kwargs): self.actual = RemoteAccess(*args, **kwargs)
#!/usr/bin/python # Demonstrates how to use the replay function to fetch the # changes made in a revision. from subvertpy.ra import RemoteAccess, Auth, get_username_provider conn = RemoteAccess("svn://svn.gnome.org/svn/gnome-specimen/trunk", auth=Auth([get_username_provider()])) class MyFileEditor: def change_prop(self, key, value): print("Change prop: %s -> %r" % (key, value)) def apply_textdelta(self, base_checksum): # This should return a function that can receive delta windows def apply_window(x): pass return apply_window def close(self): pass class MyDirEditor: def open_directory(self, *args): print("Open dir: %s (base revnum: %r)" % args) return MyDirEditor() def add_directory(self, path, copyfrom_path=None, copyfrom_rev=-1):
#!/usr/bin/python # Demonstrates how to use the replay function to fetch the # changes made in a revision. from subvertpy.ra import RemoteAccess, Auth, get_username_provider conn = RemoteAccess("svn://svn.gnome.org/svn/gnome-specimen/trunk", auth=Auth([get_username_provider()])) class MyFileEditor: def change_prop(self, key, value): print("Change prop: %s -> %r" % (key, value)) def apply_textdelta(self, base_checksum): # This should return a function that can receive delta windows def apply_window(x): pass return apply_window def close(self): pass class MyDirEditor: def open_directory(self, *args): print("Open dir: %s (base revnum: %r)" % args) return MyDirEditor()
def remote_access(self, auth: Auth) -> RemoteAccess: """Simple wrapper around subvertpy.ra.RemoteAccess creation enabling to retry the operation if a network error occurs.""" return RemoteAccess(self.remote_url, auth=auth)