def test_remote_access_retry_failure(
mocker, tmp_path, sample_repo_url, exception_to_retry
):
nb_failed_calls = SVN_RETRY_MAX_ATTEMPTS
mock_ra = mocker.patch("swh.loader.svn.svn.RemoteAccess")
remote_access = RemoteAccess(sample_repo_url, auth=Auth([get_username_provider()]))
mock_ra.side_effect = (
[exception_to_retry] * nb_failed_calls
+ [remote_access]
+ [exception_to_retry] * nb_failed_calls
+ [remote_access]
)
mock_sleep = mocker.patch.object(SvnRepo.remote_access.retry, "sleep")
with pytest.raises(type(exception_to_retry)):
SvnRepo(
sample_repo_url,
sample_repo_url,
tmp_path,
max_content_length=100000,
)
assert_sleep_calls(mock_sleep, mocker, nb_failed_calls - 1)
def accept_ssl_certificate(self, path, on_failure=None):
"""If the repository uses SSL, this method is used to determine whether
the SSL certificate can be automatically accepted.
If the cert cannot be accepted, the ``on_failure`` callback
is executed.
``on_failure`` signature::
void on_failure(e:Exception, path:str, cert:dict)
"""
self._accept_cert = {}
self._accept_on_failure = on_failure
auth = ra.Auth([
ra.get_simple_provider(),
ra.get_username_provider(),
ra.get_ssl_server_trust_prompt_provider(self._accept_trust_prompt),
])
cfg = get_config(self.config_dir)
client = SVNClient(cfg, auth)
try:
info = client.info(path)
logging.debug('SVN: Got repository information for %s: %s' %
(path, info))
except SubversionException as e:
if on_failure:
on_failure(e, path, self._accept_cert)
def __init__(self, config_dir, repopath, username=None, password=None):
super(Client, self).__init__(config_dir, repopath, username, password)
self.repopath = B(self.repopath)
self.config_dir = B(config_dir)
self._ssl_trust_prompt_cb = None
auth_providers = [
ra.get_simple_provider(),
ra.get_username_provider(),
]
if repopath.startswith('https:'):
auth_providers += [
ra.get_ssl_client_cert_file_provider(),
ra.get_ssl_client_cert_pw_file_provider(),
ra.get_ssl_server_trust_file_provider(),
ra.get_ssl_server_trust_prompt_provider(self.ssl_trust_prompt),
]
self.auth = ra.Auth(auth_providers)
if username:
self.auth.set_parameter(B('svn:auth:username'), B(username))
if password:
self.auth.set_parameter(B('svn:auth:password'), B(password))
cfg = get_config(self.config_dir)
self.client = SVNClient(cfg, auth=self.auth)
def test_checks_uuid(self):
repos_url = self.make_svn_repository('d')
dc = self.get_commit_editor(repos_url)
dc.add_dir("bp")
dc.close()
mapping = mapping_registry.get_default()()
ra = RemoteAccess(repos_url.encode("utf-8"),
auth=Auth([get_username_provider()]))
revnum = ra.get_latest_revnum()
revprops = {
SVN_REVPROP_BZR_REPOS_UUID: "otheruuid",
"svn:log": "bla",
SVN_REVPROP_BZR_ROOT: "bp",
SVN_REVPROP_BZR_MAPPING_VERSION: mapping.name,
SVN_REVPROP_BZR_BASE_REVISION: "therealbaserevid"
}
dc = TestCommitEditor(ra.get_commit_editor(revprops), ra.url, revnum)
dc.open_dir("bp").add_file("bp/la").modify()
dc.close()
repos = Repository.open(repos_url)
revmeta1 = repos._revmeta_provider.get_revision(u"bp", 1)
revmeta2 = repos._revmeta_provider.get_revision(u"bp", 2)
self.assertEquals(
mapping.revision_id_foreign_to_bzr((repos.uuid, "bp", 1)),
revmeta2.get_lhs_parent_revid(mapping, revmeta1))
def test_diff(self):
r = ra.RemoteAccess(self.repos_url,
auth=ra.Auth([ra.get_username_provider()]))
dc = self.get_commit_editor(self.repos_url)
f = dc.add_file("foo")
f.modify("foo1")
dc.close()
dc = self.get_commit_editor(self.repos_url)
f = dc.open_file("foo")
f.modify("foo2")
dc.close()
if client.api_version() < (1, 5):
self.assertRaises(NotImplementedError, self.client.diff, 1, 2,
self.repos_url, self.repos_url)
return # Skip test
(outf, errf) = self.client.diff(1, 2, self.repos_url, self.repos_url)
self.addCleanup(outf.close)
self.addCleanup(errf.close)
self.assertEqual(
"""Index: foo
===================================================================
--- foo\t(revision 1)
+++ foo\t(revision 2)
@@ -1 +1 @@
-foo1
\\ No newline at end of file
+foo2
\\ No newline at end of file
""", outf.read())
self.assertEqual("", errf.read())
def test_diff(self):
r = ra.RemoteAccess(self.repos_url,
auth=ra.Auth([ra.get_username_provider()]))
dc = self.get_commit_editor(self.repos_url)
f = dc.add_file("foo")
f.modify("foo1")
dc.close()
dc = self.get_commit_editor(self.repos_url)
f = dc.open_file("foo")
f.modify("foo2")
dc.close()
if client.api_version() < (1, 5):
self.assertRaises(NotImplementedError, self.client.diff, 1, 2,
self.repos_url, self.repos_url)
return # Skip test
(outf, errf) = self.client.diff(1, 2, self.repos_url, self.repos_url)
self.addCleanup(outf.close)
self.addCleanup(errf.close)
self.assertEqual("""Index: foo
===================================================================
--- foo\t(revision 1)
+++ foo\t(revision 2)
@@ -1 +1 @@
-foo1
\\ No newline at end of file
+foo2
\\ No newline at end of file
""", outf.read())
self.assertEqual("", errf.read())
def __init__(self, config_dir, repopath, username=None, password=None):
super(Client, self).__init__(config_dir, repopath, username, password)
self.repopath = B(self.repopath)
self.config_dir = B(config_dir)
self._ssl_trust_prompt_cb = None
auth_providers = [
ra.get_simple_provider(),
ra.get_username_provider(),
]
if repopath.startswith('https:'):
auth_providers += [
ra.get_ssl_client_cert_file_provider(),
ra.get_ssl_client_cert_pw_file_provider(),
ra.get_ssl_server_trust_file_provider(),
ra.get_ssl_server_trust_prompt_provider(self.ssl_trust_prompt),
]
self.auth = ra.Auth(auth_providers)
if username:
self.auth.set_parameter(B('svn:auth:username'), B(username))
if password:
self.auth.set_parameter(B('svn:auth:password'), B(password))
cfg = get_config(self.config_dir)
self.client = SVNClient(cfg, auth=self.auth)
def accept_ssl_certificate(self, path, on_failure=None):
"""If the repository uses SSL, this method is used to determine whether
the SSL certificate can be automatically accepted.
If the cert cannot be accepted, the ``on_failure`` callback
is executed.
``on_failure`` signature::
void on_failure(e:Exception, path:str, cert:dict)
"""
self._accept_cert = {}
self._accept_on_failure = on_failure
auth = ra.Auth([
ra.get_simple_provider(),
ra.get_username_provider(),
ra.get_ssl_server_trust_prompt_provider(self._accept_trust_prompt),
])
cfg = get_config(self.config_dir)
client = SVNClient(cfg, auth)
try:
info = client.info(path)
logging.debug('SVN: Got repository information for %s: %s' %
(path, info))
except SubversionException as e:
if on_failure:
on_failure(e, path, self._accept_cert)
def _init_client(self):
self.client_ctx = client.Client()
self.client_ctx.auth = Auth([ra.get_simple_provider(),
ra.get_username_provider(),
ra.get_ssl_client_cert_file_provider(),
ra.get_ssl_client_cert_pw_file_provider(),
ra.get_ssl_server_trust_file_provider()])
self.client_ctx.log_msg_func = self.log_message_func
def _init_client(self):
self.client_ctx = client.Client()
self.client_ctx.auth = Auth([ra.get_simple_provider(),
ra.get_username_provider(),
ra.get_ssl_client_cert_file_provider(),
ra.get_ssl_client_cert_pw_file_provider(),
ra.get_ssl_server_trust_file_provider()])
self.client_ctx.log_msg_func = self.log_message_func
def test_commit_start(self):
self.build_tree({"dc/foo": None})
self.client = client.Client(auth=ra.Auth([ra.get_username_provider()]),
log_msg_func=lambda c: "Bmessage")
self.client.add("dc/foo")
self.client.commit(["dc"])
r = ra.RemoteAccess(self.repos_url)
revprops = r.rev_proplist(1)
self.assertEqual("Bmessage", revprops["svn:log"])
def accept_ssl_certificate(self, path, on_failure=None):
"""If the repository uses SSL, this method is used to determine whether
the SSL certificate can be automatically accepted.
If the cert cannot be accepted, the ``on_failure`` callback
is executed.
``on_failure`` signature::
void on_failure(e:Exception, path:str, cert:dict)
"""
cert = {}
def _accept_trust_prompt(realm, failures, certinfo, may_save):
cert.update({
'realm': realm,
'failures': failures,
'hostname': certinfo[0],
'finger_print': certinfo[1],
'valid_from': certinfo[2],
'valid_until': certinfo[3],
'issuer_dname': certinfo[4],
})
if on_failure:
return 0, False
else:
del cert['failures']
return failures, True
auth = ra.Auth([
ra.get_simple_provider(),
ra.get_username_provider(),
ra.get_ssl_client_cert_file_provider(),
ra.get_ssl_client_cert_pw_file_provider(),
ra.get_ssl_server_trust_file_provider(),
ra.get_ssl_server_trust_prompt_provider(_accept_trust_prompt),
])
if self.username:
auth.set_parameter(AUTH_PARAM_DEFAULT_USERNAME, self.username)
if self.password:
auth.set_parameter(AUTH_PARAM_DEFAULT_PASSWORD, self.password)
cfg = get_config(self.config_dir)
client = SVNClient(cfg, auth)
try:
info = client.info(path)
logging.debug('SVN: Got repository information for %s: %s' %
(path, info))
except SubversionException as e:
if on_failure:
on_failure(e, path, cert)
return cert
def test_commit_start(self):
self.build_tree({"dc/foo": None})
self.client = client.Client(auth=ra.Auth([ra.get_username_provider()]),
log_msg_func=lambda c: "Bmessage")
self.client.add("dc/foo")
self.client.commit(["dc"])
r = ra.RemoteAccess(self.repos_url)
revprops = r.rev_proplist(1)
self.assertEqual("Bmessage", revprops["svn:log"])
def accept_ssl_certificate(self, path, on_failure=None):
"""If the repository uses SSL, this method is used to determine whether
the SSL certificate can be automatically accepted.
If the cert cannot be accepted, the ``on_failure`` callback
is executed.
``on_failure`` signature::
void on_failure(e:Exception, path:str, cert:dict)
"""
cert = {}
def _accept_trust_prompt(realm, failures, certinfo, may_save):
cert.update({
'realm': realm,
'failures': failures,
'hostname': certinfo[0],
'finger_print': certinfo[1],
'valid_from': certinfo[2],
'valid_until': certinfo[3],
'issuer_dname': certinfo[4],
})
if on_failure:
return 0, False
else:
del cert['failures']
return failures, True
auth = ra.Auth([
ra.get_simple_provider(),
ra.get_username_provider(),
ra.get_ssl_client_cert_file_provider(),
ra.get_ssl_client_cert_pw_file_provider(),
ra.get_ssl_server_trust_file_provider(),
ra.get_ssl_server_trust_prompt_provider(_accept_trust_prompt),
])
if self.username:
auth.set_parameter(AUTH_PARAM_DEFAULT_USERNAME, B(self.username))
if self.password:
auth.set_parameter(AUTH_PARAM_DEFAULT_PASSWORD, B(self.password))
cfg = get_config(self.config_dir)
client = SVNClient(cfg, auth)
try:
info = client.info(path)
logging.debug('SVN: Got repository information for %s: %s' %
(path, info))
except SubversionException as e:
if on_failure:
on_failure(e, path, cert)
return cert
def client_set_revprop(self, url, revnum, name, value):
"""Set a revision property on a repository.
:param url: URL of the repository
:param revnum: Revision number of the revision
:param name: Name of the property
:param value: Value of the property, None to remove
"""
r = ra.RemoteAccess(url, auth=Auth([ra.get_username_provider()]))
r.change_rev_prop(revnum, name, value)
def get_commit_editor(self, url, message="Test commit"):
"""Obtain a commit editor.
:param url: URL to connect to
:param message: Commit message
:return: Commit editor object
"""
ra_ctx = RemoteAccess(url.encode("utf-8"), auth=Auth([ra.get_username_provider()]))
revnum = ra_ctx.get_latest_revnum()
return TestCommitEditor(ra_ctx.get_commit_editor({"svn:log": message}), ra_ctx.url, revnum)
def client_set_revprop(self, url, revnum, name, value):
"""Set a revision property on a repository.
:param url: URL of the repository
:param revnum: Revision number of the revision
:param name: Name of the property
:param value: Value of the property, None to remove
"""
r = ra.RemoteAccess(url, auth=Auth([ra.get_username_provider()]))
r.change_rev_prop(revnum, name, value)
def accept_ssl_certificate(self, path, on_failure=None):
"""If the repository uses SSL, this method is used to determine whether
the SSL certificate can be automatically accepted.
If the cert cannot be accepted, the ``on_failure`` callback
is executed.
``on_failure`` signature::
void on_failure(e:Exception, path:str, cert:dict)
"""
cert = {}
def _accept_trust_prompt(realm, failures, certinfo, may_save):
cert.update(
{
"realm": realm,
"failures": failures,
"hostname": certinfo[0],
"finger_print": certinfo[1],
"valid_from": certinfo[2],
"valid_until": certinfo[3],
"issuer_dname": certinfo[4],
}
)
if on_failure:
return 0, False
else:
del cert["failures"]
return failures, True
auth = ra.Auth(
[
ra.get_simple_provider(),
ra.get_username_provider(),
ra.get_ssl_client_cert_file_provider(),
ra.get_ssl_client_cert_pw_file_provider(),
ra.get_ssl_server_trust_file_provider(),
ra.get_ssl_server_trust_prompt_provider(_accept_trust_prompt),
]
)
cfg = get_config(self.config_dir)
client = SVNClient(cfg, auth)
try:
info = client.info(path)
logging.debug("SVN: Got repository information for %s: %s" % (path, info))
except SubversionException as e:
if on_failure:
on_failure(e, path, cert)
return cert
def get_commit_editor(self, url, message="Test commit"):
"""Obtain a commit editor.
:param url: URL to connect to
:param message: Commit message
:return: Commit editor object
"""
ra_ctx = RemoteAccess(url.encode("utf-8"),
auth=Auth([ra.get_username_provider()]))
revnum = ra_ctx.get_latest_revnum()
return TestCommitEditor(ra_ctx.get_commit_editor({"svn:log": message}),
ra_ctx.url, revnum)
def __init__(self, config_dir, repopath, username=None, password=None):
"""Initialize the client.
Args:
config_dir (unicode):
The Subversion configuration directory.
repopath (unicode):
The path to the Subversion repository.
username (unicode, optional):
The username used to authenticate with the repository.
password (unicode, optional):
The password used to authenticate with the repository.
"""
super(Client, self).__init__(config_dir, repopath, username, password)
self.repopath = self.repopath
self.config_dir = config_dir
self._ssl_trust_prompt_cb = None
auth_providers = [
ra.get_simple_provider(),
ra.get_username_provider(),
]
if repopath.startswith('https:'):
auth_providers += [
ra.get_ssl_client_cert_file_provider(),
ra.get_ssl_client_cert_pw_file_provider(),
ra.get_ssl_server_trust_file_provider(),
ra.get_ssl_server_trust_prompt_provider(self.ssl_trust_prompt),
]
self.auth = ra.Auth(auth_providers)
self.username = None
self.password = None
if username:
self.username = username
self.auth.set_parameter(AUTH_PARAM_DEFAULT_USERNAME,
self.username)
if password:
self.password = password
self.auth.set_parameter(AUTH_PARAM_DEFAULT_PASSWORD,
self.password)
cfg = get_config(self.config_dir)
self.client = SVNClient(cfg, auth=self.auth)
def __init__(self, config_dir, repopath, username=None, password=None):
super(Client, self).__init__(config_dir, repopath, username, password)
self.repopath = B(self.repopath)
self.config_dir = B(config_dir)
auth_providers = [ra.get_simple_provider(), ra.get_username_provider()]
if repopath.startswith("https:"):
auth_providers.append(ra.get_ssl_server_trust_prompt_provider(self.ssl_trust_prompt))
self.auth = ra.Auth(auth_providers)
if username:
self.auth.set_parameter(B("svn:auth:username"), B(username))
if password:
self.auth.set_parameter(B("svn:auth:password"), B(password))
cfg = get_config(self.config_dir)
self.client = SVNClient(cfg, auth=self.auth)
def __init__(
self,
remote_url: str,
origin_url: str,
local_dirname: str,
max_content_length: int,
from_dump: bool = False,
):
self.remote_url = remote_url.rstrip("/")
self.origin_url = origin_url
self.from_dump = from_dump
auth = Auth([get_username_provider()])
# one client for update operation
self.client = client.Client(auth=auth)
if not self.remote_url.startswith("file://"):
# use redirection URL if any for remote operations
self.remote_url = self.info(self.remote_url).url
# one connection for log iteration
self.conn_log = self.remote_access(auth)
# another for replay
self.conn = self.remote_access(auth)
self.local_dirname = local_dirname
local_name = os.path.basename(self.remote_url)
self.local_url = os.path.join(self.local_dirname,
local_name).encode("utf-8")
self.uuid = self.conn.get_uuid().encode("utf-8")
self.swhreplay = replay.Replay(
conn=self.conn,
rootpath=self.local_url,
svnrepo=self,
temp_dir=local_dirname,
)
self.max_content_length = max_content_length
self.has_relative_externals = False
self.has_recursive_externals = False
self.replay_started = False
# compute root directory path from the remote repository URL, required to
# properly load the sub-tree of a repository mounted from a dump file
repos_root_url = self.info(origin_url).repos_root_url
self.root_directory = origin_url.rstrip("/").replace(
repos_root_url, "", 1)
def svn_remote(url):
# note: client only works for local path
# create an auth with stock svn providers
auth = ra.Auth([
ra.get_simple_provider(),
ra.get_username_provider(),
ra.get_ssl_client_cert_file_provider(),
ra.get_ssl_client_cert_pw_file_provider(),
ra.get_ssl_server_trust_file_provider(),
])
auth.set_parameter(subvertpy.AUTH_PARAM_DEFAULT_USERNAME,
settings.SVN_USERNAME)
auth.set_parameter(subvertpy.AUTH_PARAM_DEFAULT_PASSWORD,
settings.SVN_PASSWORD)
return ra.RemoteAccess(url, auth=auth)
def get_stock_svn_providers():
providers = [ra.get_simple_provider(),
ra.get_username_provider(),
ra.get_ssl_client_cert_file_provider(),
ra.get_ssl_client_cert_pw_file_provider(),
ra.get_ssl_server_trust_file_provider(),
]
if getattr(ra, 'get_windows_simple_provider', None):
providers.append(ra.get_windows_simple_provider())
if getattr(ra, 'get_keychain_simple_provider', None):
providers.append(ra.get_keychain_simple_provider())
if getattr(ra, 'get_windows_ssl_server_trust_provider', None):
providers.append(ra.get_windows_ssl_server_trust_provider())
return providers
def init_ra_and_client(self):
"""
Initializes the RA and client layers.
With the SWIG bindings, getting unified diffs runs the remote server
sometimes runs out of open files. It is not known whether the Subvertpy
is affected by this.
"""
def getclientstring():
return 'hgsubversion'
# TODO: handle certificate authentication, Mercurial style
def getpass(realm, username, may_save):
return self.username or username, self.password or '', False
def getuser(realm, may_save):
return self.username or '', False
providers = ra.get_platform_specific_client_providers()
providers += [
ra.get_simple_provider(),
ra.get_username_provider(),
ra.get_ssl_client_cert_file_provider(),
ra.get_ssl_client_cert_pw_file_provider(),
ra.get_ssl_server_trust_file_provider(),
ra.get_username_prompt_provider(getuser, 0),
ra.get_simple_prompt_provider(getpass, 0),
]
auth = ra.Auth(providers)
if self.username:
auth.set_parameter(subvertpy.AUTH_PARAM_DEFAULT_USERNAME,
self.username)
if self.password:
auth.set_parameter(subvertpy.AUTH_PARAM_DEFAULT_PASSWORD,
self.password)
self.remote = ra.RemoteAccess(url=self.svn_url,
client_string_func=getclientstring,
auth=auth)
self.client = client.Client()
self.client.auth = auth
def add_commit(repo_url: str, message: str, changes: List[CommitChange]) -> None:
conn = RemoteAccess(repo_url, auth=Auth([get_username_provider()]))
editor = conn.get_commit_editor({"svn:log": message})
root = editor.open_root()
for change in changes:
if change["change_type"] == CommitChangeType.Delete:
root.delete_entry(change["path"].rstrip("/"))
else:
dir_change = change["path"].endswith("/")
split_path = change["path"].rstrip("/").split("/")
for i in range(len(split_path)):
path = "/".join(split_path[0 : i + 1])
if i < len(split_path) - 1:
try:
root.add_directory(path).close()
except SubversionException:
pass
else:
if dir_change:
try:
dir = root.add_directory(path)
except SubversionException:
dir = root.open_directory(path)
if "properties" in change:
for prop, value in change["properties"].items():
dir.change_prop(prop, value)
dir.close()
else:
try:
file = root.add_file(path)
except SubversionException:
file = root.open_file(path)
if "properties" in change:
for prop, value in change["properties"].items():
file.change_prop(prop, value)
if "data" in change:
txdelta = file.apply_textdelta()
delta.send_stream(BytesIO(change["data"]), txdelta)
file.close()
root.close()
editor.close()
def main(local_url, svn_url, revision_start, revision_end, debug, cleanup):
"""Script to present how to use Replay class."""
conn = RemoteAccess(svn_url.encode("utf-8"), auth=Auth([get_username_provider()]))
os.makedirs(local_url, exist_ok=True)
rootpath = tempfile.mkdtemp(
prefix=local_url, suffix="-" + os.path.basename(svn_url)
)
rootpath = os.fsencode(rootpath)
# Do not go beyond the repository's latest revision
revision_end_max = conn.get_latest_revnum()
if revision_end == -1:
revision_end = revision_end_max
revision_end = min(revision_end, revision_end_max)
try:
replay = Replay(conn, rootpath)
for rev in range(revision_start, revision_end + 1):
contents, skipped_contents, directories = replay.compute_objects(rev)
print(
"r%s %s (%s new contents, %s new directories)"
% (
rev,
hashutil.hash_to_hex(replay.directory.hash),
len(contents) + len(skipped_contents),
len(directories),
)
)
if debug:
print("%s" % rootpath.decode("utf-8"))
finally:
if cleanup:
if os.path.exists(rootpath):
shutil.rmtree(rootpath)
def init_ra_and_client(self):
"""
Initializes the RA and client layers.
With the SWIG bindings, getting unified diffs runs the remote server
sometimes runs out of open files. It is not known whether the Subvertpy
is affected by this.
"""
def getclientstring():
return 'hgsubversion'
# TODO: handle certificate authentication, Mercurial style
def getpass(realm, username, may_save):
return self.username or username, self.password or '', False
def getuser(realm, may_save):
return self.username or '', False
providers = ra.get_platform_specific_client_providers()
providers += [
ra.get_simple_provider(),
ra.get_username_provider(),
ra.get_ssl_client_cert_file_provider(),
ra.get_ssl_client_cert_pw_file_provider(),
ra.get_ssl_server_trust_file_provider(),
ra.get_username_prompt_provider(getuser, 0),
ra.get_simple_prompt_provider(getpass, 0),
]
auth = ra.Auth(providers)
if self.username:
auth.set_parameter(subvertpy.AUTH_PARAM_DEFAULT_USERNAME, self.username)
if self.password:
auth.set_parameter(subvertpy.AUTH_PARAM_DEFAULT_PASSWORD, self.password)
self.remote = ra.RemoteAccess(url=self.svn_url,
client_string_func=getclientstring,
auth=auth)
self.client = client.Client()
self.client.auth = auth
def test_remote_access_retry_success(
mocker, tmp_path, sample_repo_url, exception_to_retry
):
nb_failed_calls = 2
mock_ra = mocker.patch("swh.loader.svn.svn.RemoteAccess")
remote_access = RemoteAccess(sample_repo_url, auth=Auth([get_username_provider()]))
mock_ra.side_effect = (
[exception_to_retry] * nb_failed_calls
+ [remote_access]
+ [exception_to_retry] * nb_failed_calls
+ [remote_access]
)
mock_sleep = mocker.patch.object(SvnRepo.remote_access.retry, "sleep")
SvnRepo(
sample_repo_url,
sample_repo_url,
tmp_path,
max_content_length=100000,
)
assert_sleep_calls(mock_sleep, mocker, nb_failed_calls)
def __init__(self, config_dir, repopath, username=None, password=None):
super(Client, self).__init__(config_dir, repopath, username, password)
self.repopath = B(self.repopath)
self.config_dir = B(config_dir)
self._ssl_trust_prompt_cb = None
auth_providers = [
ra.get_simple_provider(),
ra.get_username_provider(),
]
if repopath.startswith('https:'):
auth_providers += [
ra.get_ssl_client_cert_file_provider(),
ra.get_ssl_client_cert_pw_file_provider(),
ra.get_ssl_server_trust_file_provider(),
ra.get_ssl_server_trust_prompt_provider(self.ssl_trust_prompt),
]
self.auth = ra.Auth(auth_providers)
self.username = None
self.password = None
if username:
self.username = username
self.auth.set_parameter(AUTH_PARAM_DEFAULT_USERNAME,
B(self.username))
if password:
self.password = password
self.auth.set_parameter(AUTH_PARAM_DEFAULT_PASSWORD,
B(self.password))
cfg = get_config(self.config_dir)
self.client = SVNClient(cfg, auth=self.auth)
def init_ra_and_client(self):
"""
Initializes the RA and client layers.
With the SWIG bindings, getting unified diffs runs the remote server
sometimes runs out of open files. It is not known whether the Subvertpy
is affected by this.
"""
def getclientstring():
return 'hgsubversion'
def simple(realm, username, may_save):
return _prompt.simple(realm, username, may_save)
def username(realm, may_save):
return _prompt.username(realm, may_save)
def ssl_client_cert(realm, may_save):
return _prompt.ssl_client_cert(realm, may_save)
def ssl_client_cert_pw(realm, may_save):
return _prompt.ssl_client_cert_pw(realm, may_save)
def ssl_server_trust(realm, failures, cert_info, may_save):
creds = _prompt.ssl_server_trust(realm, failures, cert_info, may_save)
if creds is None:
# We need to reject the certificate, but subvertpy doesn't
# handle None as a return value here, and requires
# we instead return a tuple of (int, bool). Because of that,
# we return (0, False) instead.
creds = (0, False)
return creds
providers = ra.get_platform_specific_client_providers()
providers += [
ra.get_simple_provider(),
ra.get_username_provider(),
ra.get_ssl_client_cert_file_provider(),
ra.get_ssl_client_cert_pw_file_provider(),
ra.get_ssl_server_trust_file_provider(),
]
if _prompt:
providers += [
ra.get_simple_prompt_provider(simple, 2),
ra.get_username_prompt_provider(username, 2),
ra.get_ssl_client_cert_prompt_provider(ssl_client_cert, 2),
ra.get_ssl_client_cert_pw_prompt_provider(ssl_client_cert_pw, 2),
ra.get_ssl_server_trust_prompt_provider(ssl_server_trust),
]
auth = ra.Auth(providers)
if self.username:
auth.set_parameter(subvertpy.AUTH_PARAM_DEFAULT_USERNAME, self.username)
if self.password:
auth.set_parameter(subvertpy.AUTH_PARAM_DEFAULT_PASSWORD, self.password)
try:
self.remote = ra.RemoteAccess(url=self.svn_url,
client_string_func=getclientstring,
auth=auth)
except SubversionException, e:
# e.child contains a detailed error messages
msglist = []
svn_exc = e
while svn_exc:
if svn_exc.args[0]:
msglist.append(svn_exc.args[0])
svn_exc = svn_exc.child
msg = '\n'.join(msglist)
raise common.SubversionConnectionException(msg)
def setUp(self):
super(TestRemoteAccess, self).setUp()
self.repos_url = self.make_repository("d")
self.ra = ra.RemoteAccess(self.repos_url,
auth=ra.Auth([ra.get_username_provider()]))
#!/usr/bin/python
# Demonstrates how to use the replay function to fetch the
# changes made in a revision.
from subvertpy.ra import RemoteAccess, Auth, get_username_provider
conn = RemoteAccess("svn://svn.gnome.org/svn/gnome-specimen/trunk",
auth=Auth([get_username_provider()]))
class MyFileEditor:
def change_prop(self, key, value):
print("Change prop: %s -> %r" % (key, value))
def apply_textdelta(self, base_checksum):
# This should return a function that can receive delta windows
def apply_window(x):
pass
return apply_window
def close(self):
pass
class MyDirEditor:
def open_directory(self, *args):
print("Open dir: %s (base revnum: %r)" % args)
return MyDirEditor()
def init_ra_and_client(self):
"""
Initializes the RA and client layers.
With the SWIG bindings, getting unified diffs runs the remote server
sometimes runs out of open files. It is not known whether the Subvertpy
is affected by this.
"""
def getclientstring():
return 'hgsubversion'
def simple(realm, username, may_save):
return _prompt.simple(realm, username, may_save)
def username(realm, may_save):
return _prompt.username(realm, may_save)
def ssl_client_cert(realm, may_save):
return _prompt.ssl_client_cert(realm, may_save)
def ssl_client_cert_pw(realm, may_save):
return _prompt.ssl_client_cert_pw(realm, may_save)
def ssl_server_trust(realm, failures, cert_info, may_save):
creds = _prompt.ssl_server_trust(realm, failures, cert_info, may_save)
if creds is None:
# We need to reject the certificate, but subvertpy doesn't
# handle None as a return value here, and requires
# we instead return a tuple of (int, bool). Because of that,
# we return (0, False) instead.
creds = (0, False)
return creds
providers = ra.get_platform_specific_client_providers()
providers += [
ra.get_simple_provider(),
ra.get_username_provider(),
ra.get_ssl_client_cert_file_provider(),
ra.get_ssl_client_cert_pw_file_provider(),
ra.get_ssl_server_trust_file_provider(),
]
if _prompt:
providers += [
ra.get_simple_prompt_provider(simple, 2),
ra.get_username_prompt_provider(username, 2),
ra.get_ssl_client_cert_prompt_provider(ssl_client_cert, 2),
ra.get_ssl_client_cert_pw_prompt_provider(ssl_client_cert_pw, 2),
ra.get_ssl_server_trust_prompt_provider(ssl_server_trust),
]
auth = ra.Auth(providers)
if self.username:
auth.set_parameter(subvertpy.AUTH_PARAM_DEFAULT_USERNAME, self.username)
if self.password:
auth.set_parameter(subvertpy.AUTH_PARAM_DEFAULT_PASSWORD, self.password)
try:
self.remote = ra.RemoteAccess(url=self.svn_url,
client_string_func=getclientstring,
auth=auth)
except SubversionException, e:
# e.child contains a detailed error messages
msglist = []
svn_exc = e
while svn_exc:
if svn_exc.args[0]:
msglist.append(svn_exc.args[0])
svn_exc = svn_exc.child
msg = '\n'.join(msglist)
raise common.SubversionConnectionException(msg)
def setUp(self):
super(TestClient, self).setUp()
self.repos_url = self.make_client("d", "dc")
self.client = client.Client(auth=ra.Auth([ra.get_username_provider()]))
def calculate_hot_spots(jiraKey,repoUrl,end_time = time.time(),cache_update_time = time.time()):
providers = ra.get_platform_specific_client_providers()
providers += [
ra.get_simple_provider(),
ra.get_username_provider(),
ra.get_ssl_client_cert_file_provider(),
ra.get_ssl_client_cert_pw_file_provider(),
ra.get_ssl_server_trust_file_provider(),
ra.get_username_prompt_provider(getuser, 0),
ra.get_simple_prompt_provider(getpass, 0),
]
auth=ra.Auth(providers)
auth.set_parameter(subvertpy.AUTH_PARAM_DEFAULT_USERNAME, settings.username)
auth.set_parameter(subvertpy.AUTH_PARAM_DEFAULT_PASSWORD, settings.password)
conn = ra.RemoteAccess(repoUrl,auth=auth)
global bugCache
global svnCache
if jiraKey in bugCache:
bugs = bugCache[jiraKey]
else:
bugs = []
if jiraKey in svnCache:
svn_entries = svnCache[jiraKey]
else:
svn_entries = []
start_time = end_time
scores = {}
authors = {}
modified_files = []
if (len(bugs) == 0 and len(svn_entries) == 0) or time.time() > cache_update_time:
#retrieve the SVN log entries
for (changed_paths, rev, revprops, has_children) in conn.iter_log(paths=None,start=0, end=conn.get_latest_revnum(), discover_changed_paths=True):
svn_entries.append((changed_paths, rev, revprops, has_children))
#query jira for all the closed bugs
bugs = get_bugs(jiraKey)
#add to the cache dictionary
bugCache[jiraKey] = bugs
svnCache[jiraKey] = svn_entries
for (changed_paths, rev, revprops, has_children) in svn_entries:
commit_time = time.mktime(dateutil.parser.parse(revprops["svn:date"]).timetuple())
if commit_time <= end_time:
#this svn commit contains code that fixed a bug
if fixed_bug(revprops["svn:log"].decode('utf8') ,bugs):
#only consider *.java and *.js files for now
modified_files.extend([(commit_time,filename,revprops["svn:author"]) for filename in changed_paths.keys() if is_source_file(filename)])
if commit_time < start_time:
start_time = commit_time
for modified_file in modified_files:
filename = modified_file[1]
author = modified_file[2]
#as per Google's description, normalize t between 0 and 1
t = (modified_file[0]-start_time)/(end_time-start_time)
#google's magic sauce
score = 1/(1+(math.e**(-12*t+12)))
#map the score to the file
if filename not in scores:
scores[filename] = score
else:
scores[filename] = scores[filename] + score
#map the author(s) to the file
if filename not in authors:
authors[filename] = [author]
else:
authors[filename].append(author)
#convert the list of authors to a map containing the counts
for filename,authorsList in authors.items():
authors[filename]=Counter(authorsList)
sorted_scores = sorted(scores.iteritems(), key=operator.itemgetter(1))
sorted_scores.reverse()
#add the author count to the scores tuple
scoresWithAuthors =[]
for score in sorted_scores:
scoresWithAuthors.append((score[0],score[1],authors[score[0]]))
#return the top 10 hotspots
return scoresWithAuthors[:10]
#!/usr/bin/python
# Demonstrates how to do a new commit using Subvertpy
import os
from cStringIO import StringIO
from subvertpy import delta, repos
from subvertpy.ra import RemoteAccess, Auth, get_username_provider
# Create a repository
repos.create("tmprepo")
# Connect to the "remote" repository using the file transport.
# Note that a username provider needs to be provided, so that Subversion
# knows who to record as the author of new commits made over this connection.
repo_url = "file://%s" % os.path.abspath("tmprepo")
conn = RemoteAccess(repo_url, auth=Auth([get_username_provider()]))
# Simple commit that adds a directory
editor = conn.get_commit_editor({"svn:log": "Commit message"})
root = editor.open_root()
# Add a directory
dir = root.add_directory("somedir")
dir.close()
# Add and edit a file
file = root.add_file("somefile")
# Set the svn:executable attribute
file.change_prop("svn:executable", "*")
# Obtain a textdelta handler and send the new file contents
txdelta = file.apply_textdelta()
delta.send_stream(StringIO("new file contents"), txdelta)
file.close()
def setUp(self):
super(TestClient, self).setUp()
self.repos_url = self.make_client("d", "dc")
self.client = client.Client(auth=ra.Auth([ra.get_username_provider()]))
#!/usr/bin/python3
# Demonstrates how to do access the working tree using subvertpy
import os
from subvertpy import client, repos, wc
from subvertpy.ra import Auth, get_username_provider
# Create a repository
repos.create("tmprepo")
c = client.Client(auth=Auth([get_username_provider()]))
c.checkout("file://" + os.getcwd() + "/tmprepo", "tmpco", "HEAD")
w = wc.WorkingCopy(None, "tmpco")
print(w)
entry = w.entry("tmpco")
print(entry.revision)
print(entry.url)
#!/usr/bin/python
# Demonstrates how to do access the working tree using subvertpy
import os
from subvertpy import client, repos, wc
from subvertpy.ra import Auth, get_username_provider
# Create a repository
repos.create("tmprepo")
c = client.Client(auth=Auth([get_username_provider()]))
c.checkout("file://" + os.getcwd() + "/tmprepo", "tmpco", "HEAD")
w = wc.WorkingCopy(None, "tmpco")
print(w)
entry = w.entry("tmpco")
print(entry.revision)
print(entry.url)
#!/usr/bin/python
# Demonstrates how to use the replay function to fetch the
# changes made in a revision.
from subvertpy.ra import RemoteAccess, Auth, get_username_provider
conn = RemoteAccess("svn://svn.gnome.org/svn/gnome-specimen/trunk",
auth=Auth([get_username_provider()]))
class MyFileEditor:
def change_prop(self, key, value):
print("Change prop: %s -> %r" % (key, value))
def apply_textdelta(self, base_checksum):
# This should return a function that can receive delta windows
def apply_window(x):
pass
return apply_window
def close(self):
pass
class MyDirEditor:
def open_directory(self, *args):
print("Open dir: %s (base revnum: %r)" % args)
return MyDirEditor()
def add_directory(self, path, copyfrom_path=None, copyfrom_rev=-1):
def setUp(self):
super(TestRemoteAccess, self).setUp()
self.repos_url = self.make_repository("d")
self.ra = ra.RemoteAccess(self.repos_url,
auth=ra.Auth([ra.get_username_provider()]))
#!/usr/bin/python3
# Demonstrates how to do a new commit using Subvertpy
import os
from io import BytesIO
from subvertpy import delta, repos
from subvertpy.ra import RemoteAccess, Auth, get_username_provider
# Create a repository
repos.create("tmprepo")
# Connect to the "remote" repository using the file transport.
# Note that a username provider needs to be provided, so that Subversion
# knows who to record as the author of new commits made over this connection.
repo_url = "file://%s" % os.path.abspath("tmprepo")
conn = RemoteAccess(repo_url, auth=Auth([get_username_provider()]))
# Simple commit that adds a directory
editor = conn.get_commit_editor({"svn:log": "Commit message"})
root = editor.open_root()
# Add a directory
dir = root.add_directory("somedir")
dir.close()
# Add and edit a file
file = root.add_file("somefile")
# Set the svn:executable attribute
file.change_prop("svn:executable", "*")
# Obtain a textdelta handler and send the new file contents
txdelta = file.apply_textdelta()
delta.send_stream(BytesIO(b"new file contents"), txdelta)
file.close()
