def test_remote_access_retry_failure(
mocker, tmp_path, sample_repo_url, exception_to_retry
):
nb_failed_calls = SVN_RETRY_MAX_ATTEMPTS
mock_ra = mocker.patch("swh.loader.svn.svn.RemoteAccess")
remote_access = RemoteAccess(sample_repo_url, auth=Auth([get_username_provider()]))
mock_ra.side_effect = (
[exception_to_retry] * nb_failed_calls
+ [remote_access]
+ [exception_to_retry] * nb_failed_calls
+ [remote_access]
)
mock_sleep = mocker.patch.object(SvnRepo.remote_access.retry, "sleep")
with pytest.raises(type(exception_to_retry)):
SvnRepo(
sample_repo_url,
sample_repo_url,
tmp_path,
max_content_length=100000,
)
assert_sleep_calls(mock_sleep, mocker, nb_failed_calls - 1)
def test_checks_uuid(self):
repos_url = self.make_svn_repository('d')
dc = self.get_commit_editor(repos_url)
dc.add_dir("bp")
dc.close()
mapping = mapping_registry.get_default()()
ra = RemoteAccess(repos_url.encode("utf-8"),
auth=Auth([get_username_provider()]))
revnum = ra.get_latest_revnum()
revprops = {
SVN_REVPROP_BZR_REPOS_UUID: "otheruuid",
"svn:log": "bla",
SVN_REVPROP_BZR_ROOT: "bp",
SVN_REVPROP_BZR_MAPPING_VERSION: mapping.name,
SVN_REVPROP_BZR_BASE_REVISION: "therealbaserevid"
}
dc = TestCommitEditor(ra.get_commit_editor(revprops), ra.url, revnum)
dc.open_dir("bp").add_file("bp/la").modify()
dc.close()
repos = Repository.open(repos_url)
revmeta1 = repos._revmeta_provider.get_revision(u"bp", 1)
revmeta2 = repos._revmeta_provider.get_revision(u"bp", 2)
self.assertEquals(
mapping.revision_id_foreign_to_bzr((repos.uuid, "bp", 1)),
revmeta2.get_lhs_parent_revid(mapping, revmeta1))
def _init_client(self):
self.client_ctx = client.Client()
self.client_ctx.auth = Auth([ra.get_simple_provider(),
ra.get_username_provider(),
ra.get_ssl_client_cert_file_provider(),
ra.get_ssl_client_cert_pw_file_provider(),
ra.get_ssl_server_trust_file_provider()])
self.client_ctx.log_msg_func = self.log_message_func
def client_set_revprop(self, url, revnum, name, value):
"""Set a revision property on a repository.
:param url: URL of the repository
:param revnum: Revision number of the revision
:param name: Name of the property
:param value: Value of the property, None to remove
"""
r = ra.RemoteAccess(url, auth=Auth([ra.get_username_provider()]))
r.change_rev_prop(revnum, name, value)
def get_commit_editor(self, url, message="Test commit"):
"""Obtain a commit editor.
:param url: URL to connect to
:param message: Commit message
:return: Commit editor object
"""
ra_ctx = RemoteAccess(url.encode("utf-8"),
auth=Auth([ra.get_username_provider()]))
revnum = ra_ctx.get_latest_revnum()
return TestCommitEditor(ra_ctx.get_commit_editor({"svn:log": message}),
ra_ctx.url, revnum)
def __init__(
self,
remote_url: str,
origin_url: str,
local_dirname: str,
max_content_length: int,
from_dump: bool = False,
):
self.remote_url = remote_url.rstrip("/")
self.origin_url = origin_url
self.from_dump = from_dump
auth = Auth([get_username_provider()])
# one client for update operation
self.client = client.Client(auth=auth)
if not self.remote_url.startswith("file://"):
# use redirection URL if any for remote operations
self.remote_url = self.info(self.remote_url).url
# one connection for log iteration
self.conn_log = self.remote_access(auth)
# another for replay
self.conn = self.remote_access(auth)
self.local_dirname = local_dirname
local_name = os.path.basename(self.remote_url)
self.local_url = os.path.join(self.local_dirname,
local_name).encode("utf-8")
self.uuid = self.conn.get_uuid().encode("utf-8")
self.swhreplay = replay.Replay(
conn=self.conn,
rootpath=self.local_url,
svnrepo=self,
temp_dir=local_dirname,
)
self.max_content_length = max_content_length
self.has_relative_externals = False
self.has_recursive_externals = False
self.replay_started = False
# compute root directory path from the remote repository URL, required to
# properly load the sub-tree of a repository mounted from a dump file
repos_root_url = self.info(origin_url).repos_root_url
self.root_directory = origin_url.rstrip("/").replace(
repos_root_url, "", 1)
def add_commit(repo_url: str, message: str, changes: List[CommitChange]) -> None:
conn = RemoteAccess(repo_url, auth=Auth([get_username_provider()]))
editor = conn.get_commit_editor({"svn:log": message})
root = editor.open_root()
for change in changes:
if change["change_type"] == CommitChangeType.Delete:
root.delete_entry(change["path"].rstrip("/"))
else:
dir_change = change["path"].endswith("/")
split_path = change["path"].rstrip("/").split("/")
for i in range(len(split_path)):
path = "/".join(split_path[0 : i + 1])
if i < len(split_path) - 1:
try:
root.add_directory(path).close()
except SubversionException:
pass
else:
if dir_change:
try:
dir = root.add_directory(path)
except SubversionException:
dir = root.open_directory(path)
if "properties" in change:
for prop, value in change["properties"].items():
dir.change_prop(prop, value)
dir.close()
else:
try:
file = root.add_file(path)
except SubversionException:
file = root.open_file(path)
if "properties" in change:
for prop, value in change["properties"].items():
file.change_prop(prop, value)
if "data" in change:
txdelta = file.apply_textdelta()
delta.send_stream(BytesIO(change["data"]), txdelta)
file.close()
root.close()
editor.close()
def main(local_url, svn_url, revision_start, revision_end, debug, cleanup):
"""Script to present how to use Replay class."""
conn = RemoteAccess(svn_url.encode("utf-8"), auth=Auth([get_username_provider()]))
os.makedirs(local_url, exist_ok=True)
rootpath = tempfile.mkdtemp(
prefix=local_url, suffix="-" + os.path.basename(svn_url)
)
rootpath = os.fsencode(rootpath)
# Do not go beyond the repository's latest revision
revision_end_max = conn.get_latest_revnum()
if revision_end == -1:
revision_end = revision_end_max
revision_end = min(revision_end, revision_end_max)
try:
replay = Replay(conn, rootpath)
for rev in range(revision_start, revision_end + 1):
contents, skipped_contents, directories = replay.compute_objects(rev)
print(
"r%s %s (%s new contents, %s new directories)"
% (
rev,
hashutil.hash_to_hex(replay.directory.hash),
len(contents) + len(skipped_contents),
len(directories),
)
)
if debug:
print("%s" % rootpath.decode("utf-8"))
finally:
if cleanup:
if os.path.exists(rootpath):
shutil.rmtree(rootpath)
def test_remote_access_retry_success(
mocker, tmp_path, sample_repo_url, exception_to_retry
):
nb_failed_calls = 2
mock_ra = mocker.patch("swh.loader.svn.svn.RemoteAccess")
remote_access = RemoteAccess(sample_repo_url, auth=Auth([get_username_provider()]))
mock_ra.side_effect = (
[exception_to_retry] * nb_failed_calls
+ [remote_access]
+ [exception_to_retry] * nb_failed_calls
+ [remote_access]
)
mock_sleep = mocker.patch.object(SvnRepo.remote_access.retry, "sleep")
SvnRepo(
sample_repo_url,
sample_repo_url,
tmp_path,
max_content_length=100000,
)
assert_sleep_calls(mock_sleep, mocker, nb_failed_calls)
#!/usr/bin/python
# Demonstrates how to use the replay function to fetch the
# changes made in a revision.
from subvertpy.ra import RemoteAccess, Auth, get_username_provider
conn = RemoteAccess("svn://svn.gnome.org/svn/gnome-specimen/trunk",
auth=Auth([get_username_provider()]))
class MyFileEditor:
def change_prop(self, key, value):
print("Change prop: %s -> %r" % (key, value))
def apply_textdelta(self, base_checksum):
# This should return a function that can receive delta windows
def apply_window(x):
pass
return apply_window
def close(self):
pass
class MyDirEditor:
def open_directory(self, *args):
print("Open dir: %s (base revnum: %r)" % args)
return MyDirEditor()
def add_directory(self, path, copyfrom_path=None, copyfrom_rev=-1):
#!/usr/bin/python
# Demonstrates how to do a new commit using Subvertpy
import os
from cStringIO import StringIO
from subvertpy import delta, repos
from subvertpy.ra import RemoteAccess, Auth, get_username_provider
# Create a repository
repos.create("tmprepo")
# Connect to the "remote" repository using the file transport.
# Note that a username provider needs to be provided, so that Subversion
# knows who to record as the author of new commits made over this connection.
repo_url = "file://%s" % os.path.abspath("tmprepo")
conn = RemoteAccess(repo_url, auth=Auth([get_username_provider()]))
# Simple commit that adds a directory
editor = conn.get_commit_editor({"svn:log": "Commit message"})
root = editor.open_root()
# Add a directory
dir = root.add_directory("somedir")
dir.close()
# Add and edit a file
file = root.add_file("somefile")
# Set the svn:executable attribute
file.change_prop("svn:executable", "*")
# Obtain a textdelta handler and send the new file contents
txdelta = file.apply_textdelta()
delta.send_stream(StringIO("new file contents"), txdelta)
file.close()
#!/usr/bin/python
# Demonstrates how to do access the working tree using subvertpy
import os
from subvertpy import client, repos, wc
from subvertpy.ra import Auth, get_username_provider
# Create a repository
repos.create("tmprepo")
c = client.Client(auth=Auth([get_username_provider()]))
c.checkout("file://" + os.getcwd() + "/tmprepo", "tmpco", "HEAD")
w = wc.WorkingCopy(None, "tmpco")
print(w)
entry = w.entry("tmpco")
print(entry.revision)
print(entry.url)