def test_inet_server_responds_to_sighup(self): t = transport.get_transport('memory:///') content = b'a' * 1024 * 1024 t.put_bytes('bigfile', content) factory = server.BzrServerFactory() # Override stdin/stdout so that we can inject our own handles client_read, server_write = self.create_file_pipes() server_read, client_write = self.create_file_pipes() factory._get_stdin_stdout = lambda: (server_read, server_write) factory.set_up(t, None, None, inet=True, timeout=4.0) self.addCleanup(factory.tear_down) started = threading.Event() stopped = threading.Event() def serving(): started.set() factory.smart_server.serve() stopped.set() server_thread = threading.Thread(target=serving) server_thread.start() started.wait() client_medium = medium.SmartSimplePipesClientMedium(client_read, client_write, 'base') client_client = client._SmartClient(client_medium) resp, response_handler = client_client.call_expecting_body(b'get', b'bigfile') signals._sighup_handler(SIGHUP, None) self.assertTrue(factory.smart_server.finished) # We can still finish reading the file content, but more than that, and # the file is closed. v = response_handler.read_body_bytes() if v != content: self.fail('Got the wrong content back, expected 1M "a"') stopped.wait() server_thread.join()
def get_transport_for_thread(base): thread_transports = getattr(_transport_store, 'transports', None) if thread_transports is None: thread_transports = _transport_store.transports = {} if base in thread_transports: return thread_transports[base] transport = get_transport(base) thread_transports[base] = transport return transport
def process(self, url): base, path = urlutils.split(url) t = transport.get_transport(base) try: # TODO: We should probably look into using some part of # blocking_timeout to decide when to stop trying to read # content content = t.get_bytes(path) except (errors.TransportError, errors.NoSuchFile): return False return True
def sprout(self, url, revision_id=None, force_new_repo=False, recurse='down', possible_transports=None, accelerator_tree=None, hardlink=False, stacked=False, source_branch=None, create_tree_if_local=True): from breezy.repository import InterRepository from breezy.transport.local import LocalTransport from breezy.transport import get_transport target_transport = get_transport(url, possible_transports) target_transport.ensure_base() cloning_format = self.cloning_metadir() # Create/update the result branch result = cloning_format.initialize_on_transport(target_transport) source_branch = self.open_branch() source_repository = self.find_repository() try: result_repo = result.find_repository() except errors.NoRepositoryPresent: result_repo = result.create_repository() target_is_empty = True else: target_is_empty = None # Unknown if stacked: raise errors.IncompatibleRepositories(source_repository, result_repo) interrepo = InterRepository.get(source_repository, result_repo) interrepo.fetch(revision_id=revision_id) result_branch = source_branch.sprout(result, revision_id=revision_id, repository=result_repo) if (create_tree_if_local and isinstance(target_transport, LocalTransport) and (result_repo is None or result_repo.make_working_trees())): wt = result.create_workingtree(accelerator_tree=accelerator_tree, hardlink=hardlink, from_branch=result_branch) wt.lock_write() try: if wt.path2id('') is None: try: wt.set_root_id(self.open_workingtree.get_root_id()) except errors.NoWorkingTree: pass finally: wt.unlock() return result
def test_export_loom_as_branch(self): tree = self.get_multi_threaded() tree.branch.controldir.root_transport.mkdir('root') root_path = tree.branch.controldir.root_transport.local_abspath('root') repo = self.make_repository('root', shared=True) repo.set_make_working_trees(False) root_transport = get_transport('root') tree.branch.export_threads(root_transport) self.assertRaises(errors.NoWorkingTree, WorkingTree.open, root_transport.local_abspath('thread1')) export_branch = Branch.open_from_transport( root_transport.clone('thread1')) self.assertEqual(b'thread1-id', export_branch.last_revision())
def write_manifest_to_transport( location, base_branch, possible_transports=None): """Write a manifest to disk. :param location: Location to write to :param base_branch: Recipe base branch """ child_transport = _mod_transport.get_transport( location, possible_transports=possible_transports) base_transport = child_transport.clone('..') base_transport.create_prefix() basename = base_transport.relpath(child_transport.base) base_transport.put_bytes(basename, str(base_branch).encode())
def open_branch_containing( url: str, possible_transports: Optional[List[Transport]] = None, probers: Optional[List[Prober]] = None, ) -> Tuple[Branch, str]: """Open a branch by URL.""" try: transport = get_transport(url, possible_transports=possible_transports) dir, subpath = ControlDir.open_containing_from_transport(transport, probers) # type: ignore return dir.open_branch(), subpath except Exception as e: converted = _convert_exception(url, e) if converted is not None: raise converted raise e
def test_log_transport(self): base_transport = self.get_transport('') logging_transport = transport.get_transport('log+' + base_transport.base) # operations such as mkdir are logged mutter('where are you?') logging_transport.mkdir('subdir') log = self.get_log() # GZ 2017-05-24: Used to expect abspath logged, logger needs fixing. self.assertContainsRe(log, r'mkdir subdir') self.assertContainsRe(log, ' --> None') # they have the expected effect self.assertTrue(logging_transport.has('subdir')) # and they operate on the underlying transport self.assertTrue(base_transport.has('subdir'))
def make_controldir(self, relpath, format=None): try: url = self.get_url(relpath) segments = url.split('/') if segments and segments[-1] not in ('', '.'): parent = '/'.join(segments[:-1]) t = transport.get_transport(parent) try: t.mkdir(segments[-1]) except FileExists: pass if format is None: format = self.repository_format._matchingcontroldir return format.initialize(url) except UninitializableFormat: raise TestSkipped("Format %s is not initializable." % format)
def get_recipe_from_location(location, possible_transports=None): """Open a recipe as a file-like object from a URL. :param location: The recipe location :param possible_transports: Possible transports to use :return: Tuple with basename and file-like object """ m = launchpad_recipe_re.match(location) if m: (username, recipe_name) = m.groups() text = get_recipe_from_launchpad(username, recipe_name, location) return (recipe_name, StringIO(text)) child_transport = _mod_transport.get_transport( location, possible_transports=possible_transports) recipe_transport = child_transport.clone('..') basename = recipe_transport.relpath(child_transport.base) return basename, recipe_transport.get(basename)
def run(self, url, private_token=None, name=None, no_check=False): from breezy import ui from .hoster import store_gitlab_token if name is None: try: name = urlutils.parse_url(url)[3].split('.')[-2] except (ValueError, IndexError): raise errors.CommandError( 'please specify a site name with --name') if private_token is None: note("Please visit %s to obtain a private token.", urlutils.join(url, "-/profile/personal_access_tokens")) private_token = ui.ui_factory.get_password(u'Private token') if not no_check: from breezy.transport import get_transport from .hoster import GitLab GitLab(get_transport(url), private_token=private_token) store_gitlab_token(name=name, url=url, private_token=private_token)
def make_branch_and_tree(self, relpath): bzrdir_format = self.workingtree_format.get_controldir_for_branch() made_control = self.make_controldir(relpath, format=bzrdir_format) made_control.create_repository() b = made_control.create_branch() if getattr(self, 'repo_is_remote', False): # If the repo is remote, then we just create a local lightweight # checkout # XXX: This duplicates a lot of Branch.create_checkout, but we know # we want a) lightweight, and b) a specific WT format. We also # know that nothing should already exist, etc. t = transport.get_transport(relpath) t.ensure_base() wt_dir = bzrdir_format.initialize_on_transport(t) branch_ref = wt_dir.set_branch_reference(b) wt = wt_dir.create_workingtree(None, from_branch=branch_ref) else: wt = self.workingtree_format.initialize(made_control) return wt
def find_repository(self, _ignore_branch_path=False): """Open the repository associated with this ControlDir. :return: instance of SvnRepository. """ from .repository import SvnRepository from .transport import get_svn_ra_transport if self.root_url != self.root_transport.base: transport = get_transport( self.root_url, possible_transports=[self.root_transport]) svn_transport = get_svn_ra_transport(transport) else: transport = self.root_transport svn_transport = self.svn_transport if _ignore_branch_path: return SvnRepository(self, transport, svn_transport) else: return SvnRepository(self, transport, svn_transport, self._branch_path)
def sproutOrSkip(self, from_bzrdir, to_url, revision_id=None, force_new_repo=False, accelerator_tree=None, create_tree_if_local=True): """Sprout from_bzrdir into to_url, or raise TestSkipped. A simple wrapper for from_bzrdir.sprout that translates NotLocalUrl into TestSkipped. Returns the newly sprouted bzrdir. """ to_transport = transport.get_transport(to_url) if not isinstance(to_transport, LocalTransport): raise TestSkipped('Cannot sprout to remote bzrdirs.') target = from_bzrdir.sprout(to_url, revision_id=revision_id, force_new_repo=force_new_repo, possible_transports=[to_transport], accelerator_tree=accelerator_tree, create_tree_if_local=create_tree_if_local) return target
def open_branch( url: str, possible_transports: Optional[List[Transport]] = None, probers: Optional[List[Prober]] = None, name: str = None, ) -> Branch: """Open a branch by URL.""" url, params = urlutils.split_segment_parameters(url) if name is None: try: name = urlutils.unquote(params["branch"]) except KeyError: name = None try: transport = get_transport(url, possible_transports=possible_transports) dir = ControlDir.open_from_transport(transport, probers) return dir.open_branch(name=name) except Exception as e: converted = _convert_exception(url, e) if converted is not None: raise converted raise e
def _create_lightweight_checkout(self, to_location, revision_id=None): """Create a new lightweight checkout of this branch. :param to_location: URL of location to create the checkout in. :param revision_id: Tip of the checkout. :return: WorkingTree object of the checkout. """ from .workingtree import ( SvnCheckout, SvnWorkingTreeDirFormat, update_wc, ) if revision_id is None or revision_id == self.last_revision(): bp = self.get_branch_path() uuid = self.repository.uuid revnum = self.get_revnum() else: (uuid, bp, revnum), mapping = self.lookup_bzr_revision_id(revision_id) transport = get_transport(to_location) transport.ensure_base() to_path = transport.local_abspath(".") svn_url, readonly = bzr_to_svn_url( urlutils.join(self.repository.base, bp)) wc.ensure_adm(to_path.encode("utf-8"), uuid, svn_url, bzr_to_svn_url(self.repository.base)[0], revnum) with wc.Adm(None, to_path.encode("utf-8"), write_lock=True) as adm: conn = self.repository.svn_transport.connections.get(svn_url) try: update_wc(adm, to_path.encode("utf-8"), conn, svn_url, revnum) finally: if not conn.busy: self.repository.svn_transport.add_connection(conn) dir = SvnCheckout(transport, SvnWorkingTreeDirFormat()) return dir.open_workingtree()
def sprout(self, url, revision_id=None, force_new_repo=False, recurse='down', possible_transports=None, accelerator_tree=None, hardlink=False, stacked=False, source_branch=None, create_tree_if_local=True): from breezy.repository import InterRepository from breezy.transport.local import LocalTransport relpath = self._determine_relpath(None) if relpath == u"": guessed_layout = self.find_repository().get_guessed_layout() if guessed_layout is not None and not guessed_layout.is_branch( u""): trace.warning( 'Cloning Subversion repository as branch. ' 'To import the individual branches in the repository, ' 'use "bzr svn-import".') target_transport = get_transport(url, possible_transports) target_transport.ensure_base() require_colocated = ("branch" in target_transport.get_segment_parameters()) cloning_format = self.cloning_metadir( require_colocated=require_colocated) # Create/update the result branch result = cloning_format.initialize_on_transport(target_transport) source_repository = self.find_repository() if force_new_repo: result_repo = result.create_repository() target_is_empty = True else: try: result_repo = result.find_repository() except errors.NoRepositoryPresent: result_repo = result.create_repository() target_is_empty = True else: target_is_empty = None # Unknown if stacked: raise UnstackableBranchFormat(self._format.get_branch_format(), self.root_transport.base) interrepo = InterRepository.get(source_repository, result_repo) try: source_branch = self.open_branch() except errors.NotBranchError: source_branch = None project = None mapping = None else: project = source_branch.project mapping = source_branch.mapping interrepo.fetch(revision_id=revision_id, project=project, mapping=mapping, target_is_empty=target_is_empty, exclude_non_mainline=False) if source_branch is not None: if revision_id is None: revision_id = source_branch.last_revision() result_branch = source_branch.sprout(result, revision_id=revision_id, repository=result_repo) interbranch = InterBranch.get(source_branch, result_branch) interbranch.fetch(stop_revision=revision_id, exclude_non_mainline=False) # For the tags else: result_branch = result.create_branch() if (create_tree_if_local and isinstance(target_transport, LocalTransport) and (result_repo is None or result_repo.make_working_trees())): result.create_workingtree(accelerator_tree=accelerator_tree, hardlink=hardlink, from_branch=result_branch) return result
import threading import time import Queue import simplejson from breezy import ( errors, transport, urlutils, ) # This code will be doing multi-threaded requests against breezy.transport # code. We want to make sure to load everything ahead of time, so we don't get # lazy-import failures _ = transport.get_transport('http://example.com') class RequestDescription(object): """Describes info about a request.""" def __init__(self, descrip_dict): self.thread = descrip_dict.get('thread', '1') self.relpath = descrip_dict['relpath'] class RequestWorker(object): """Process requests in a worker thread.""" _timer = time.time
def __init__(self, source_repos, output_url, layout=None, create_shared_repo=True, working_trees=False, all=False, format=None, filter_branch=None, keep=False, incremental=False, to_revnum=None, prefix=None, colocated=False, remember_parent=True): """Convert a Subversion repository and its' branches to a Bazaar repository. :param source_repos: Subversion repository :param output_url: URL to write Bazaar repository to. :param layout: Repository layout (object) to use :param create_shared_repo: Whether to create a shared Bazaar repository :param working_trees: Whether to create working trees :param all: Whether old revisions, even those not part of any existing branches, should be imported. :param format: Format to use :param remember_parent: Remember parent branch location """ assert not all or create_shared_repo if format is None: self._format = controldir.format_registry.make_controldir( 'default') else: self._format = format self.dirs = {} self.to_transport = get_transport(output_url) try: self.to_transport.mkdir('.') except FileExists: pass if layout is not None: source_repos.set_layout(layout) else: layout = source_repos.get_layout() try: target_dir = controldir.ControlDir.open_from_transport( self.to_transport) except NotBranchError: target_dir = None else: format = target_dir._format if colocated is None: colocated = getattr(format, "colocated_branches", False) if target_dir is None and (colocated or create_shared_repo): target_dir = self.get_dir(prefix, prefix) if create_shared_repo: try: target_repos = target_dir.open_repository() target_repos_is_empty = False # FIXME: Call Repository.is_empty() ? if not layout.is_branch(u"") and not target_repos.is_shared( ) and not colocated: raise BzrError("Repository %r is not shared." % target_repos) except NoRepositoryPresent: target_repos = target_dir.create_repository(shared=True) target_repos_is_empty = True target_repos.set_make_working_trees(working_trees) else: target_repos = None target_repos_is_empty = False with source_repos.lock_read(): if incremental and target_repos is not None: from_revnum = get_latest_svn_import_revision( target_repos, source_repos.uuid) else: from_revnum = 0 if to_revnum is None: to_revnum = source_repos.get_latest_revnum() if to_revnum < from_revnum: return mapping = source_repos.get_mapping() existing_branches = {} deleted = set() it = source_repos._revmeta_provider.iter_all_changes( layout, mapping.is_branch_or_tag, to_revnum, from_revnum, prefix=prefix) if create_shared_repo: revfinder = FetchRevisionFinder(source_repos, target_repos, target_repos_is_empty) revmetas = [] else: revmetas = None if all: heads = None else: heads = set() with ui.ui_factory.nested_progress_bar() as pb: for kind, item in it: if kind == "revision": pb.update("finding branches", to_revnum - item.metarev.revnum, to_revnum - from_revnum) if (not item.metarev.branch_path in existing_branches and layout.is_branch(item.metarev.branch_path) and not contains_parent_path( deleted, item.metarev.branch_path)): existing_branches[ item.metarev.branch_path] = SvnBranch( source_repos, None, item.metarev.branch_path, revnum=item.metarev.revnum, _skip_check=True, mapping=mapping) if heads is not None: heads.add(item) if revmetas is not None: revmetas.append(item) elif kind == "delete": (path, revnum) = item deleted.add(path) if create_shared_repo: if not InterFromSvnToInventoryRepository.is_compatible( source_repos, target_repos): raise IncompatibleRepositories(source_repos, target_repos) inter = InterFromSvnToInventoryRepository.get( source_repos, target_repos) self._fetch_to_shared_repo(inter, prefix, from_revnum, revmetas, revfinder, mapping, heads) if not keep: self._remove_branches(deleted, existing_branches.keys()) existing_branches = existing_branches.values() if filter_branch is not None: existing_branches = filter(filter_branch, existing_branches) self._create_branches(existing_branches, prefix, create_shared_repo, working_trees, colocated, remember_parent) if target_repos is not None: put_latest_svn_import_revision(target_repos, source_repos.uuid, to_revnum)
def _redirected_to(self, source, target): # We do accept redirections return transport.get_transport(target)
def get_remote_cache_transport(): return get_transport(get_cache_dir())