def test_inet_server_responds_to_sighup(self):
t = transport.get_transport('memory:///')
content = b'a' * 1024 * 1024
t.put_bytes('bigfile', content)
factory = server.BzrServerFactory()
# Override stdin/stdout so that we can inject our own handles
client_read, server_write = self.create_file_pipes()
server_read, client_write = self.create_file_pipes()
factory._get_stdin_stdout = lambda: (server_read, server_write)
factory.set_up(t, None, None, inet=True, timeout=4.0)
self.addCleanup(factory.tear_down)
started = threading.Event()
stopped = threading.Event()
def serving():
started.set()
factory.smart_server.serve()
stopped.set()
server_thread = threading.Thread(target=serving)
server_thread.start()
started.wait()
client_medium = medium.SmartSimplePipesClientMedium(client_read,
client_write, 'base')
client_client = client._SmartClient(client_medium)
resp, response_handler = client_client.call_expecting_body(b'get',
b'bigfile')
signals._sighup_handler(SIGHUP, None)
self.assertTrue(factory.smart_server.finished)
# We can still finish reading the file content, but more than that, and
# the file is closed.
v = response_handler.read_body_bytes()
if v != content:
self.fail('Got the wrong content back, expected 1M "a"')
stopped.wait()
server_thread.join()
def get_transport_for_thread(base):
thread_transports = getattr(_transport_store, 'transports', None)
if thread_transports is None:
thread_transports = _transport_store.transports = {}
if base in thread_transports:
return thread_transports[base]
transport = get_transport(base)
thread_transports[base] = transport
return transport
def process(self, url):
base, path = urlutils.split(url)
t = transport.get_transport(base)
try:
# TODO: We should probably look into using some part of
# blocking_timeout to decide when to stop trying to read
# content
content = t.get_bytes(path)
except (errors.TransportError, errors.NoSuchFile):
return False
return True
def sprout(self,
url,
revision_id=None,
force_new_repo=False,
recurse='down',
possible_transports=None,
accelerator_tree=None,
hardlink=False,
stacked=False,
source_branch=None,
create_tree_if_local=True):
from breezy.repository import InterRepository
from breezy.transport.local import LocalTransport
from breezy.transport import get_transport
target_transport = get_transport(url, possible_transports)
target_transport.ensure_base()
cloning_format = self.cloning_metadir()
# Create/update the result branch
result = cloning_format.initialize_on_transport(target_transport)
source_branch = self.open_branch()
source_repository = self.find_repository()
try:
result_repo = result.find_repository()
except errors.NoRepositoryPresent:
result_repo = result.create_repository()
target_is_empty = True
else:
target_is_empty = None # Unknown
if stacked:
raise errors.IncompatibleRepositories(source_repository,
result_repo)
interrepo = InterRepository.get(source_repository, result_repo)
interrepo.fetch(revision_id=revision_id)
result_branch = source_branch.sprout(result,
revision_id=revision_id,
repository=result_repo)
if (create_tree_if_local
and isinstance(target_transport, LocalTransport)
and (result_repo is None or result_repo.make_working_trees())):
wt = result.create_workingtree(accelerator_tree=accelerator_tree,
hardlink=hardlink,
from_branch=result_branch)
wt.lock_write()
try:
if wt.path2id('') is None:
try:
wt.set_root_id(self.open_workingtree.get_root_id())
except errors.NoWorkingTree:
pass
finally:
wt.unlock()
return result
def test_export_loom_as_branch(self):
tree = self.get_multi_threaded()
tree.branch.controldir.root_transport.mkdir('root')
root_path = tree.branch.controldir.root_transport.local_abspath('root')
repo = self.make_repository('root', shared=True)
repo.set_make_working_trees(False)
root_transport = get_transport('root')
tree.branch.export_threads(root_transport)
self.assertRaises(errors.NoWorkingTree, WorkingTree.open,
root_transport.local_abspath('thread1'))
export_branch = Branch.open_from_transport(
root_transport.clone('thread1'))
self.assertEqual(b'thread1-id', export_branch.last_revision())
def write_manifest_to_transport(
location, base_branch, possible_transports=None):
"""Write a manifest to disk.
:param location: Location to write to
:param base_branch: Recipe base branch
"""
child_transport = _mod_transport.get_transport(
location, possible_transports=possible_transports)
base_transport = child_transport.clone('..')
base_transport.create_prefix()
basename = base_transport.relpath(child_transport.base)
base_transport.put_bytes(basename, str(base_branch).encode())
def open_branch_containing(
url: str,
possible_transports: Optional[List[Transport]] = None,
probers: Optional[List[Prober]] = None,
) -> Tuple[Branch, str]:
"""Open a branch by URL."""
try:
transport = get_transport(url, possible_transports=possible_transports)
dir, subpath = ControlDir.open_containing_from_transport(transport, probers) # type: ignore
return dir.open_branch(), subpath
except Exception as e:
converted = _convert_exception(url, e)
if converted is not None:
raise converted
raise e
def test_log_transport(self):
base_transport = self.get_transport('')
logging_transport = transport.get_transport('log+' +
base_transport.base)
# operations such as mkdir are logged
mutter('where are you?')
logging_transport.mkdir('subdir')
log = self.get_log()
# GZ 2017-05-24: Used to expect abspath logged, logger needs fixing.
self.assertContainsRe(log, r'mkdir subdir')
self.assertContainsRe(log, ' --> None')
# they have the expected effect
self.assertTrue(logging_transport.has('subdir'))
# and they operate on the underlying transport
self.assertTrue(base_transport.has('subdir'))
def make_controldir(self, relpath, format=None):
try:
url = self.get_url(relpath)
segments = url.split('/')
if segments and segments[-1] not in ('', '.'):
parent = '/'.join(segments[:-1])
t = transport.get_transport(parent)
try:
t.mkdir(segments[-1])
except FileExists:
pass
if format is None:
format = self.repository_format._matchingcontroldir
return format.initialize(url)
except UninitializableFormat:
raise TestSkipped("Format %s is not initializable." % format)
def get_recipe_from_location(location, possible_transports=None):
"""Open a recipe as a file-like object from a URL.
:param location: The recipe location
:param possible_transports: Possible transports to use
:return: Tuple with basename and file-like object
"""
m = launchpad_recipe_re.match(location)
if m:
(username, recipe_name) = m.groups()
text = get_recipe_from_launchpad(username, recipe_name, location)
return (recipe_name, StringIO(text))
child_transport = _mod_transport.get_transport(
location, possible_transports=possible_transports)
recipe_transport = child_transport.clone('..')
basename = recipe_transport.relpath(child_transport.base)
return basename, recipe_transport.get(basename)
def run(self, url, private_token=None, name=None, no_check=False):
from breezy import ui
from .hoster import store_gitlab_token
if name is None:
try:
name = urlutils.parse_url(url)[3].split('.')[-2]
except (ValueError, IndexError):
raise errors.CommandError(
'please specify a site name with --name')
if private_token is None:
note("Please visit %s to obtain a private token.",
urlutils.join(url, "-/profile/personal_access_tokens"))
private_token = ui.ui_factory.get_password(u'Private token')
if not no_check:
from breezy.transport import get_transport
from .hoster import GitLab
GitLab(get_transport(url), private_token=private_token)
store_gitlab_token(name=name, url=url, private_token=private_token)
def make_branch_and_tree(self, relpath):
bzrdir_format = self.workingtree_format.get_controldir_for_branch()
made_control = self.make_controldir(relpath, format=bzrdir_format)
made_control.create_repository()
b = made_control.create_branch()
if getattr(self, 'repo_is_remote', False):
# If the repo is remote, then we just create a local lightweight
# checkout
# XXX: This duplicates a lot of Branch.create_checkout, but we know
# we want a) lightweight, and b) a specific WT format. We also
# know that nothing should already exist, etc.
t = transport.get_transport(relpath)
t.ensure_base()
wt_dir = bzrdir_format.initialize_on_transport(t)
branch_ref = wt_dir.set_branch_reference(b)
wt = wt_dir.create_workingtree(None, from_branch=branch_ref)
else:
wt = self.workingtree_format.initialize(made_control)
return wt
def find_repository(self, _ignore_branch_path=False):
"""Open the repository associated with this ControlDir.
:return: instance of SvnRepository.
"""
from .repository import SvnRepository
from .transport import get_svn_ra_transport
if self.root_url != self.root_transport.base:
transport = get_transport(
self.root_url, possible_transports=[self.root_transport])
svn_transport = get_svn_ra_transport(transport)
else:
transport = self.root_transport
svn_transport = self.svn_transport
if _ignore_branch_path:
return SvnRepository(self, transport, svn_transport)
else:
return SvnRepository(self, transport, svn_transport,
self._branch_path)
def sproutOrSkip(self,
from_bzrdir,
to_url,
revision_id=None,
force_new_repo=False,
accelerator_tree=None,
create_tree_if_local=True):
"""Sprout from_bzrdir into to_url, or raise TestSkipped.
A simple wrapper for from_bzrdir.sprout that translates NotLocalUrl into
TestSkipped. Returns the newly sprouted bzrdir.
"""
to_transport = transport.get_transport(to_url)
if not isinstance(to_transport, LocalTransport):
raise TestSkipped('Cannot sprout to remote bzrdirs.')
target = from_bzrdir.sprout(to_url,
revision_id=revision_id,
force_new_repo=force_new_repo,
possible_transports=[to_transport],
accelerator_tree=accelerator_tree,
create_tree_if_local=create_tree_if_local)
return target
def open_branch(
url: str,
possible_transports: Optional[List[Transport]] = None,
probers: Optional[List[Prober]] = None,
name: str = None,
) -> Branch:
"""Open a branch by URL."""
url, params = urlutils.split_segment_parameters(url)
if name is None:
try:
name = urlutils.unquote(params["branch"])
except KeyError:
name = None
try:
transport = get_transport(url, possible_transports=possible_transports)
dir = ControlDir.open_from_transport(transport, probers)
return dir.open_branch(name=name)
except Exception as e:
converted = _convert_exception(url, e)
if converted is not None:
raise converted
raise e
def _create_lightweight_checkout(self, to_location, revision_id=None):
"""Create a new lightweight checkout of this branch.
:param to_location: URL of location to create the checkout in.
:param revision_id: Tip of the checkout.
:return: WorkingTree object of the checkout.
"""
from .workingtree import (
SvnCheckout,
SvnWorkingTreeDirFormat,
update_wc,
)
if revision_id is None or revision_id == self.last_revision():
bp = self.get_branch_path()
uuid = self.repository.uuid
revnum = self.get_revnum()
else:
(uuid, bp,
revnum), mapping = self.lookup_bzr_revision_id(revision_id)
transport = get_transport(to_location)
transport.ensure_base()
to_path = transport.local_abspath(".")
svn_url, readonly = bzr_to_svn_url(
urlutils.join(self.repository.base, bp))
wc.ensure_adm(to_path.encode("utf-8"), uuid, svn_url,
bzr_to_svn_url(self.repository.base)[0], revnum)
with wc.Adm(None, to_path.encode("utf-8"), write_lock=True) as adm:
conn = self.repository.svn_transport.connections.get(svn_url)
try:
update_wc(adm, to_path.encode("utf-8"), conn, svn_url, revnum)
finally:
if not conn.busy:
self.repository.svn_transport.add_connection(conn)
dir = SvnCheckout(transport, SvnWorkingTreeDirFormat())
return dir.open_workingtree()
def sprout(self,
url,
revision_id=None,
force_new_repo=False,
recurse='down',
possible_transports=None,
accelerator_tree=None,
hardlink=False,
stacked=False,
source_branch=None,
create_tree_if_local=True):
from breezy.repository import InterRepository
from breezy.transport.local import LocalTransport
relpath = self._determine_relpath(None)
if relpath == u"":
guessed_layout = self.find_repository().get_guessed_layout()
if guessed_layout is not None and not guessed_layout.is_branch(
u""):
trace.warning(
'Cloning Subversion repository as branch. '
'To import the individual branches in the repository, '
'use "bzr svn-import".')
target_transport = get_transport(url, possible_transports)
target_transport.ensure_base()
require_colocated = ("branch"
in target_transport.get_segment_parameters())
cloning_format = self.cloning_metadir(
require_colocated=require_colocated)
# Create/update the result branch
result = cloning_format.initialize_on_transport(target_transport)
source_repository = self.find_repository()
if force_new_repo:
result_repo = result.create_repository()
target_is_empty = True
else:
try:
result_repo = result.find_repository()
except errors.NoRepositoryPresent:
result_repo = result.create_repository()
target_is_empty = True
else:
target_is_empty = None # Unknown
if stacked:
raise UnstackableBranchFormat(self._format.get_branch_format(),
self.root_transport.base)
interrepo = InterRepository.get(source_repository, result_repo)
try:
source_branch = self.open_branch()
except errors.NotBranchError:
source_branch = None
project = None
mapping = None
else:
project = source_branch.project
mapping = source_branch.mapping
interrepo.fetch(revision_id=revision_id,
project=project,
mapping=mapping,
target_is_empty=target_is_empty,
exclude_non_mainline=False)
if source_branch is not None:
if revision_id is None:
revision_id = source_branch.last_revision()
result_branch = source_branch.sprout(result,
revision_id=revision_id,
repository=result_repo)
interbranch = InterBranch.get(source_branch, result_branch)
interbranch.fetch(stop_revision=revision_id,
exclude_non_mainline=False) # For the tags
else:
result_branch = result.create_branch()
if (create_tree_if_local
and isinstance(target_transport, LocalTransport)
and (result_repo is None or result_repo.make_working_trees())):
result.create_workingtree(accelerator_tree=accelerator_tree,
hardlink=hardlink,
from_branch=result_branch)
return result
import threading
import time
import Queue
import simplejson
from breezy import (
errors,
transport,
urlutils,
)
# This code will be doing multi-threaded requests against breezy.transport
# code. We want to make sure to load everything ahead of time, so we don't get
# lazy-import failures
_ = transport.get_transport('http://example.com')
class RequestDescription(object):
"""Describes info about a request."""
def __init__(self, descrip_dict):
self.thread = descrip_dict.get('thread', '1')
self.relpath = descrip_dict['relpath']
class RequestWorker(object):
"""Process requests in a worker thread."""
_timer = time.time
def __init__(self,
source_repos,
output_url,
layout=None,
create_shared_repo=True,
working_trees=False,
all=False,
format=None,
filter_branch=None,
keep=False,
incremental=False,
to_revnum=None,
prefix=None,
colocated=False,
remember_parent=True):
"""Convert a Subversion repository and its' branches to a
Bazaar repository.
:param source_repos: Subversion repository
:param output_url: URL to write Bazaar repository to.
:param layout: Repository layout (object) to use
:param create_shared_repo: Whether to create a shared Bazaar
repository
:param working_trees: Whether to create working trees
:param all: Whether old revisions, even those not part of any
existing branches, should be imported.
:param format: Format to use
:param remember_parent: Remember parent branch location
"""
assert not all or create_shared_repo
if format is None:
self._format = controldir.format_registry.make_controldir(
'default')
else:
self._format = format
self.dirs = {}
self.to_transport = get_transport(output_url)
try:
self.to_transport.mkdir('.')
except FileExists:
pass
if layout is not None:
source_repos.set_layout(layout)
else:
layout = source_repos.get_layout()
try:
target_dir = controldir.ControlDir.open_from_transport(
self.to_transport)
except NotBranchError:
target_dir = None
else:
format = target_dir._format
if colocated is None:
colocated = getattr(format, "colocated_branches", False)
if target_dir is None and (colocated or create_shared_repo):
target_dir = self.get_dir(prefix, prefix)
if create_shared_repo:
try:
target_repos = target_dir.open_repository()
target_repos_is_empty = False # FIXME: Call Repository.is_empty() ?
if not layout.is_branch(u"") and not target_repos.is_shared(
) and not colocated:
raise BzrError("Repository %r is not shared." %
target_repos)
except NoRepositoryPresent:
target_repos = target_dir.create_repository(shared=True)
target_repos_is_empty = True
target_repos.set_make_working_trees(working_trees)
else:
target_repos = None
target_repos_is_empty = False
with source_repos.lock_read():
if incremental and target_repos is not None:
from_revnum = get_latest_svn_import_revision(
target_repos, source_repos.uuid)
else:
from_revnum = 0
if to_revnum is None:
to_revnum = source_repos.get_latest_revnum()
if to_revnum < from_revnum:
return
mapping = source_repos.get_mapping()
existing_branches = {}
deleted = set()
it = source_repos._revmeta_provider.iter_all_changes(
layout,
mapping.is_branch_or_tag,
to_revnum,
from_revnum,
prefix=prefix)
if create_shared_repo:
revfinder = FetchRevisionFinder(source_repos, target_repos,
target_repos_is_empty)
revmetas = []
else:
revmetas = None
if all:
heads = None
else:
heads = set()
with ui.ui_factory.nested_progress_bar() as pb:
for kind, item in it:
if kind == "revision":
pb.update("finding branches",
to_revnum - item.metarev.revnum,
to_revnum - from_revnum)
if (not item.metarev.branch_path in existing_branches
and layout.is_branch(item.metarev.branch_path)
and not contains_parent_path(
deleted, item.metarev.branch_path)):
existing_branches[
item.metarev.branch_path] = SvnBranch(
source_repos,
None,
item.metarev.branch_path,
revnum=item.metarev.revnum,
_skip_check=True,
mapping=mapping)
if heads is not None:
heads.add(item)
if revmetas is not None:
revmetas.append(item)
elif kind == "delete":
(path, revnum) = item
deleted.add(path)
if create_shared_repo:
if not InterFromSvnToInventoryRepository.is_compatible(
source_repos, target_repos):
raise IncompatibleRepositories(source_repos, target_repos)
inter = InterFromSvnToInventoryRepository.get(
source_repos, target_repos)
self._fetch_to_shared_repo(inter, prefix, from_revnum,
revmetas, revfinder, mapping, heads)
if not keep:
self._remove_branches(deleted, existing_branches.keys())
existing_branches = existing_branches.values()
if filter_branch is not None:
existing_branches = filter(filter_branch, existing_branches)
self._create_branches(existing_branches, prefix,
create_shared_repo, working_trees, colocated,
remember_parent)
if target_repos is not None:
put_latest_svn_import_revision(target_repos, source_repos.uuid,
to_revnum)
def _redirected_to(self, source, target):
# We do accept redirections
return transport.get_transport(target)
def get_remote_cache_transport():
return get_transport(get_cache_dir())
