def fetch(self, import_path, dest, rev=None):
match, url_info = self._matcher(import_path)
pkg = GoRemoteLibrary.remote_package_path(self.root(import_path), import_path)
archive_url = match.expand(url_info.url_format).format(rev=url_info.rev(rev), pkg=pkg)
try:
archiver = archiver_for_path(archive_url)
except ValueError:
raise self.FetchError("Don't know how to unpack archive at url {}".format(archive_url))
with self._fetch(archive_url) as archive:
if url_info.strip_level == 0:
archiver.extract(archive, dest)
else:
with temporary_dir() as scratch:
archiver.extract(archive, scratch)
for dirpath, dirnames, filenames in os.walk(scratch, topdown=True):
if dirpath != scratch:
relpath = os.path.relpath(dirpath, scratch)
relpath_components = relpath.split(os.sep)
if len(relpath_components) == url_info.strip_level and (dirnames or filenames):
for path in dirnames + filenames:
src = os.path.join(dirpath, path)
dst = os.path.join(dest, path)
shutil.move(src, dst)
del dirnames[:] # Stops the walk.
def _map_fetched_remote_source(self, go_remote_lib, gopath, all_known_remote_libs, resolved_remote_libs, undeclared_deps):
for remote_import_path in self._get_remote_import_paths(go_remote_lib.import_path, gopath=gopath):
fetcher = self._get_fetcher(remote_import_path)
remote_root = fetcher.root()
spec_path = os.path.join(go_remote_lib.target_base, remote_root)
package_path = GoRemoteLibrary.remote_package_path(remote_root, remote_import_path)
target_name = package_path or os.path.basename(remote_root)
address = Address(spec_path, target_name)
if not any(address == lib.address for lib in all_known_remote_libs):
try:
# If we've already resolved a package from this remote root, its ok to define an
# implicit synthetic remote target for all other packages in the same remote root.
same_remote_libs = [lib for lib in all_known_remote_libs if spec_path == lib.address.spec_path]
implicit_ok = any(same_remote_libs)
# If we're creating a synthetic remote target, we should pin it to the same
# revision as the rest of the library.
rev = None
if implicit_ok:
rev = same_remote_libs[0].rev
remote_lib = self._resolve(go_remote_lib, address, package_path, rev, implicit_ok)
resolved_remote_libs.add(remote_lib)
all_known_remote_libs.add(remote_lib)
except self.UndeclaredRemoteLibError as e:
undeclared_deps[go_remote_lib].add((remote_import_path, e.address))
self.context.build_graph.inject_dependency(go_remote_lib.address, address)
def _map_fetched_remote_source(self, go_remote_lib, gopath, all_known_remote_libs, resolved_remote_libs, undeclared_deps):
for remote_import_path in self._get_remote_import_paths(go_remote_lib.import_path, gopath=gopath):
fetcher = self._get_fetcher(remote_import_path)
remote_root = fetcher.root()
spec_path = os.path.join(go_remote_lib.target_base, remote_root)
package_path = GoRemoteLibrary.remote_package_path(remote_root, remote_import_path)
target_name = package_path or os.path.basename(remote_root)
address = Address(spec_path, target_name)
if not any(address == lib.address for lib in all_known_remote_libs):
try:
# If we've already resolved a package from this remote root, its ok to define an
# implicit synthetic remote target for all other packages in the same remote root.
same_remote_libs = [lib for lib in all_known_remote_libs if spec_path == lib.address.spec_path]
implicit_ok = any(same_remote_libs)
# If we're creating a synthetic remote target, we should pin it to the same
# revision as the rest of the library.
rev = None
if implicit_ok:
rev = same_remote_libs[0].rev
remote_lib = self._resolve(go_remote_lib, address, package_path, rev, implicit_ok)
resolved_remote_libs.add(remote_lib)
all_known_remote_libs.add(remote_lib)
except self.UndeclaredRemoteLibError as e:
undeclared_deps[go_remote_lib].add((remote_import_path, e.address))
self.context.build_graph.inject_dependency(go_remote_lib.address, address)
def _generate_missing(self, gopath, local_address, import_listing, visited):
target_type = GoBinary if import_listing.pkg_name == 'main' else GoLibrary
existing = self._build_graph.get_target(local_address)
if not existing:
self._build_graph.inject_synthetic_target(address=local_address, target_type=target_type)
elif existing and not isinstance(existing, target_type):
raise self.WrongLocalSourceTargetTypeError('{} should be a {}'
.format(existing, target_type.__name__))
for import_path in import_listing.all_imports:
if not self._import_oracle.is_go_internal_import(import_path):
if import_path not in visited:
if self._import_oracle.is_remote_import(import_path):
remote_root = self._fetcher_factory.get_fetcher(import_path).root()
remote_pkg_path = GoRemoteLibrary.remote_package_path(remote_root, import_path)
name = remote_pkg_path or os.path.basename(import_path)
address = Address(os.path.join(self._remote_source_root, remote_root), name)
try:
self._build_graph.inject_address_closure(address)
except AddressLookupError:
if not self._generate_remotes:
raise self.NewRemoteEncounteredButRemotesNotAllowedError(
'Cannot generate dependency for remote import path {}'.format(import_path))
self._build_graph.inject_synthetic_target(address=address,
target_type=GoRemoteLibrary,
pkg=remote_pkg_path)
else:
# Recurse on local targets.
address = Address(os.path.join(self._local_source_root, import_path),
os.path.basename(import_path))
deps = self._list_deps(gopath, address)
self._generate_missing(gopath, address, deps, visited)
visited[import_path] = address
dependency_address = visited[import_path]
self._build_graph.inject_dependency(local_address, dependency_address)
def _map_fetched_remote_source(self, go_remote_lib, gopath,
all_known_remote_libs, resolved_remote_libs,
undeclared_deps, import_root_map):
# See if we've computed the remote import paths for this rev of this lib in a previous run.
remote_import_paths_cache = os.path.join(os.path.dirname(gopath),
'remote_import_paths.txt')
if os.path.exists(remote_import_paths_cache):
with open(remote_import_paths_cache, 'r') as fp:
remote_import_paths = [
line.decode('utf8').strip() for line in fp.readlines()
]
else:
remote_import_paths = self._get_remote_import_paths(
go_remote_lib.import_path, gopath=gopath)
with safe_concurrent_creation(
remote_import_paths_cache) as safe_path:
with open(safe_path, 'w') as fp:
for path in remote_import_paths:
fp.write('{}\n'.format(path).encode('utf8'))
for remote_import_path in remote_import_paths:
remote_root = import_root_map.get(remote_import_path)
if remote_root is None:
fetcher = self._get_fetcher(remote_import_path)
remote_root = fetcher.root()
import_root_map[remote_import_path] = remote_root
spec_path = os.path.join(go_remote_lib.target_base, remote_root)
package_path = GoRemoteLibrary.remote_package_path(
remote_root, remote_import_path)
target_name = package_path or os.path.basename(remote_root)
address = Address(spec_path, target_name)
if not any(address == lib.address
for lib in all_known_remote_libs):
try:
# If we've already resolved a package from this remote root, its ok to define an
# implicit synthetic remote target for all other packages in the same remote root.
same_remote_libs = [
lib for lib in all_known_remote_libs
if spec_path == lib.address.spec_path
]
implicit_ok = any(same_remote_libs)
# If we're creating a synthetic remote target, we should pin it to the same
# revision as the rest of the library.
rev = None
if implicit_ok:
rev = same_remote_libs[0].rev
remote_lib = self._resolve(go_remote_lib, address,
package_path, rev, implicit_ok)
resolved_remote_libs.add(remote_lib)
all_known_remote_libs.add(remote_lib)
except self.UndeclaredRemoteLibError as e:
undeclared_deps[go_remote_lib].add(
(remote_import_path, e.address))
self.context.build_graph.inject_dependency(go_remote_lib.address,
address)
def _generate_missing(self, gopath, local_address, name, import_paths, visited):
target_type = GoBinary if name == 'main' else GoLibrary
existing = self._build_graph.get_target(local_address)
if not existing:
self._build_graph.inject_synthetic_target(address=local_address, target_type=target_type)
elif existing and not isinstance(existing, target_type):
raise self.WrongLocalSourceTargetTypeError('{} should be a {}'
.format(existing, target_type.__name__))
for import_path in import_paths:
if import_path not in self._go_stdlib:
if import_path not in visited:
fetcher = self._fetchers.maybe_get_fetcher(import_path)
if fetcher:
remote_root = fetcher.root(import_path)
remote_pkg_path = GoRemoteLibrary.remote_package_path(remote_root, import_path)
name = remote_pkg_path or os.path.basename(import_path)
address = Address(os.path.join(self._remote_source_root, remote_root), name)
found = self._build_graph.get_target(address)
if not found:
if not self._generate_remotes:
raise self.NewRemoteEncounteredButRemotesNotAllowedError(
'Cannot generate dependency for remote import path {}'.format(import_path))
self._build_graph.inject_synthetic_target(address=address,
target_type=GoRemoteLibrary,
pkg=remote_pkg_path)
else:
# Recurse on local targets.
address = Address(os.path.join(self._local_source_root, import_path),
os.path.basename(import_path))
name, import_paths = self._list_deps(gopath, address)
self._generate_missing(gopath, address, name, import_paths, visited)
visited[import_path] = address
dependency_address = visited[import_path]
self._build_graph.inject_dependency(local_address, dependency_address)
def _map_fetched_remote_source(self, go_remote_lib, gopath,
all_known_addresses, resolved_remote_libs,
undeclared_deps):
for remote_import_path in self._get_remote_import_paths(
go_remote_lib.import_path, gopath=gopath):
fetcher = self._get_fetcher(remote_import_path)
remote_root = fetcher.root()
spec_path = os.path.join(go_remote_lib.target_base, remote_root)
package_path = GoRemoteLibrary.remote_package_path(
remote_root, remote_import_path)
target_name = package_path or os.path.basename(remote_root)
address = Address(spec_path, target_name)
if address not in all_known_addresses:
try:
# If we've already resolved a package from this remote root, its ok to define an
# implicit synthetic remote target for all other packages in the same remote root.
implicit_ok = any(spec_path == a.spec_path
for a in all_known_addresses)
remote_lib = self._resolve(go_remote_lib, address,
package_path, implicit_ok)
resolved_remote_libs.add(remote_lib)
all_known_addresses.add(address)
except self.UndeclaredRemoteLibError as e:
undeclared_deps[go_remote_lib].add(
(remote_import_path, e.address))
self.context.build_graph.inject_dependency(go_remote_lib.address,
address)
def fetch(self, dest, rev=None):
pkg = GoRemoteLibrary.remote_package_path(self.root(), self.import_path)
archive_url = self._url_info.url_format.format(rev=rev or self._url_info.default_rev,
pkg=pkg, import_prefix=self.root())
try:
self._fetch(archive_url, self._url_info.strip_level, dest)
except FetchError as e:
# Modify the message to add more information, then reraise with the original traceback.
e.add_message_prefix('Error while fetching import {}: '.format(self.import_path))
raise
def fetch(self, dest, rev=None):
pkg = GoRemoteLibrary.remote_package_path(self.root(),
self.import_path)
archive_url = self._url_info.url_format.format(
rev=rev or self._url_info.default_rev,
pkg=pkg,
import_prefix=self.root())
try:
self._fetch(archive_url, self._url_info.strip_level, dest)
except FetchError as e:
# Modify the message to add more information, then reraise with the original traceback.
e.add_message_prefix('Error while fetching import {}: '.format(
self.import_path))
raise
def _map_fetched_remote_source(self, go_remote_lib, gopath, all_known_remote_libs,
resolved_remote_libs, undeclared_deps, import_root_map):
# See if we've computed the remote import paths for this rev of this lib in a previous run.
remote_import_paths_cache = os.path.join(os.path.dirname(gopath), 'remote_import_paths.txt')
if os.path.exists(remote_import_paths_cache):
with open(remote_import_paths_cache, 'r') as fp:
remote_import_paths = [line.decode('utf8').strip() for line in fp.readlines()]
else:
remote_import_paths = self._get_remote_import_paths(go_remote_lib.import_path,
gopath=gopath)
with safe_concurrent_creation(remote_import_paths_cache) as safe_path:
with open(safe_path, 'w') as fp:
for path in remote_import_paths:
fp.write('{}\n'.format(path).encode('utf8'))
for remote_import_path in remote_import_paths:
remote_root = import_root_map.get(remote_import_path)
if remote_root is None:
fetcher = self._get_fetcher(remote_import_path)
remote_root = fetcher.root()
import_root_map[remote_import_path] = remote_root
spec_path = os.path.join(go_remote_lib.target_base, remote_root)
package_path = GoRemoteLibrary.remote_package_path(remote_root, remote_import_path)
target_name = package_path or os.path.basename(remote_root)
address = Address(spec_path, target_name)
if not any(address == lib.address for lib in all_known_remote_libs):
try:
# If we've already resolved a package from this remote root, its ok to define an
# implicit synthetic remote target for all other packages in the same remote root.
same_remote_libs = [lib for lib in all_known_remote_libs
if spec_path == lib.address.spec_path]
implicit_ok = any(same_remote_libs)
# If we're creating a synthetic remote target, we should pin it to the same
# revision as the rest of the library.
rev = None
if implicit_ok:
rev = same_remote_libs[0].rev
remote_lib = self._resolve(go_remote_lib, address, package_path, rev, implicit_ok)
resolved_remote_libs.add(remote_lib)
all_known_remote_libs.add(remote_lib)
except self.UndeclaredRemoteLibError as e:
undeclared_deps[go_remote_lib].add((remote_import_path, e.address))
self.context.build_graph.inject_dependency(go_remote_lib.address, address)
def _generate_missing(self, gopath, local_address, import_listing,
visited):
target_type = GoBinary if import_listing.pkg_name == "main" else GoLibrary
existing = self._build_graph.get_target(local_address)
if not existing:
self._build_graph.inject_synthetic_target(address=local_address,
target_type=target_type)
elif existing and not isinstance(existing, target_type):
raise self.WrongLocalSourceTargetTypeError(
"{} should be a {}".format(existing, target_type.__name__))
for import_path in import_listing.all_imports:
if not self._import_oracle.is_go_internal_import(import_path):
if import_path not in visited:
if self._import_oracle.is_remote_import(import_path):
remote_root = self._fetcher_factory.get_fetcher(
import_path).root()
remote_pkg_path = GoRemoteLibrary.remote_package_path(
remote_root, import_path)
name = remote_pkg_path or os.path.basename(import_path)
address = Address(
os.path.join(self._remote_source_root,
remote_root), name)
try:
self._build_graph.inject_address_closure(address)
except AddressLookupError:
if not self._generate_remotes:
raise self.NewRemoteEncounteredButRemotesNotAllowedError(
f"Cannot generate dependency for remote import path {import_path}"
)
self._build_graph.inject_synthetic_target(
address=address,
target_type=GoRemoteLibrary,
pkg=remote_pkg_path)
else:
# Recurse on local targets.
address = Address(
os.path.join(self._local_source_root, import_path),
os.path.basename(import_path),
)
deps = self._list_deps(gopath, address)
self._generate_missing(gopath, address, deps, visited)
visited[import_path] = address
dependency_address = visited[import_path]
self._build_graph.inject_dependency(local_address,
dependency_address)
def _generate_missing(self, gopath, local_address, name, import_paths,
visited):
target_type = GoBinary if name == 'main' else GoLibrary
existing = self._build_graph.get_target(local_address)
if not existing:
self._build_graph.inject_synthetic_target(address=local_address,
target_type=target_type)
elif existing and not isinstance(existing, target_type):
raise self.WrongLocalSourceTargetTypeError(
'{} should be a {}'.format(existing, target_type.__name__))
for import_path in import_paths:
if import_path not in self._go_stdlib:
if import_path not in visited:
fetcher = self._fetchers.maybe_get_fetcher(import_path)
if fetcher:
remote_root = fetcher.root(import_path)
remote_pkg_path = GoRemoteLibrary.remote_package_path(
remote_root, import_path)
name = remote_pkg_path or os.path.basename(import_path)
address = Address(
os.path.join(self._remote_source_root,
remote_root), name)
found = self._build_graph.get_target(address)
if not found:
if not self._generate_remotes:
raise self.NewRemoteEncounteredButRemotesNotAllowedError(
'Cannot generate dependency for remote import path {}'
.format(import_path))
self._build_graph.inject_synthetic_target(
address=address,
target_type=GoRemoteLibrary,
pkg=remote_pkg_path)
else:
# Recurse on local targets.
address = Address(
os.path.join(self._local_source_root, import_path),
os.path.basename(import_path))
name, import_paths = self._list_deps(gopath, address)
self._generate_missing(gopath, address, name,
import_paths, visited)
visited[import_path] = address
dependency_address = visited[import_path]
self._build_graph.inject_dependency(local_address,
dependency_address)
def _map_fetched_remote_source(self, go_remote_lib, gopath, all_known_addresses, resolved_remote_libs, undeclared_deps):
for remote_import_path in self._get_remote_import_paths(go_remote_lib.import_path, gopath=gopath):
fetcher = self._get_fetcher(remote_import_path)
remote_root = fetcher.root()
spec_path = os.path.join(go_remote_lib.target_base, remote_root)
package_path = GoRemoteLibrary.remote_package_path(remote_root, remote_import_path)
target_name = package_path or os.path.basename(remote_root)
address = Address(spec_path, target_name)
if address not in all_known_addresses:
try:
# If we've already resolved a package from this remote root, its ok to define an
# implicit synthetic remote target for all other packages in the same remote root.
implicit_ok = any(spec_path == a.spec_path for a in all_known_addresses)
remote_lib = self._resolve(go_remote_lib, address, package_path, implicit_ok)
resolved_remote_libs.add(remote_lib)
all_known_addresses.add(address)
except self.UndeclaredRemoteLibError as e:
undeclared_deps[go_remote_lib].add((remote_import_path, e.address))
self.context.build_graph.inject_dependency(go_remote_lib.address, address)
def _transitive_download_remote_libs(self,
go_remote_libs,
all_known_addresses=None):
"""Recursively attempt to resolve / download all remote transitive deps of go_remote_libs.
Returns a dict<GoRemoteLibrary, set<tuple<str, Address>>>, which maps a go remote library to a
set of unresolved remote dependencies, each dependency expressed as a tuple containing the
the import path of the dependency and the expected target address. If all transitive
dependencies were successfully resolved, returns an empty dict.
Downloads as many invalidated transitive dependencies as possible, and returns as many
undeclared dependencies as possible. However, because the dependencies of a remote library
can only be determined _after_ it has been downloaded, a transitive dependency of an undeclared
remote library will never be detected.
Because go_remote_libraries do not declare dependencies (rather, they are inferred), injects
all successfully resolved transitive dependencies into the build graph.
"""
if not go_remote_libs:
return {}
all_known_addresses = all_known_addresses or set()
all_known_addresses.update(lib.address for lib in go_remote_libs)
resolved_remote_libs = set()
undeclared_deps = defaultdict(set)
go_remote_lib_src = self.context.products.get_data('go_remote_lib_src')
with self.invalidated(go_remote_libs) as invalidation_check:
for vt in invalidation_check.all_vts:
go_remote_lib = vt.target
gopath = vt.results_dir
fetcher = self._get_fetcher(go_remote_lib.import_path)
if not vt.valid:
root = fetcher.root(go_remote_lib.import_path)
fetch_dir = os.path.join(self.workdir, 'fetches')
root_dir = os.path.join(fetch_dir, root)
# Only fetch each remote root once.
if not os.path.exists(root_dir):
with temporary_dir() as tmp_fetch_root:
fetcher.fetch(go_remote_lib.import_path,
dest=tmp_fetch_root,
rev=go_remote_lib.rev)
safe_mkdir(root_dir)
for path in os.listdir(tmp_fetch_root):
shutil.move(os.path.join(tmp_fetch_root, path),
os.path.join(root_dir, path))
# TODO(John Sirois): Circle back and get get rid of this symlink tree.
# GoWorkspaceTask will further symlink a single package from the tree below into a
# target's workspace when it could just be linking from the fetch_dir. The only thing
# standing in the way is a determination of what we want to artifact cache. If we don't
# want to cache fetched zips, linking straight from the fetch_dir works simply. Otherwise
# thought needs to be applied to using the artifact cache directly or synthesizing a
# canonical owner target for the fetched files that 'child' targets (subpackages) can
# depend on and share the fetch from.
dest_dir = os.path.join(gopath, 'src', root)
# We may have been `invalidate`d and not `clean-all`ed so we need a new empty symlink
# chroot to avoid collision; thus `clean=True`.
safe_mkdir(dest_dir, clean=True)
for path in os.listdir(root_dir):
os.symlink(os.path.join(root_dir, path),
os.path.join(dest_dir, path))
# Map the fetched remote sources.
pkg = go_remote_lib.import_path
go_remote_lib_src[go_remote_lib] = os.path.join(
gopath, 'src', pkg)
for remote_import_path in self._get_remote_import_paths(
pkg, gopath=gopath):
fetcher = self._get_fetcher(remote_import_path)
remote_root = fetcher.root(remote_import_path)
spec_path = os.path.join(go_remote_lib.target_base,
remote_root)
package_path = GoRemoteLibrary.remote_package_path(
remote_root, remote_import_path)
target_name = package_path or os.path.basename(remote_root)
address = Address(spec_path, target_name)
if address not in all_known_addresses:
try:
# If we've already resolved a package from this remote root, its ok to define an
# implicit synthetic remote target for all other packages in the same remote root.
implicit_ok = any(spec_path == a.spec_path
for a in all_known_addresses)
remote_lib = self._resolve(go_remote_lib, address,
package_path,
implicit_ok)
resolved_remote_libs.add(remote_lib)
all_known_addresses.add(address)
except self.UndeclaredRemoteLibError as e:
undeclared_deps[go_remote_lib].add(
(remote_import_path, e.address))
self.context.build_graph.inject_dependency(
go_remote_lib.address, address)
# Recurse after the invalidated block, so the libraries we downloaded are now "valid"
# and thus we don't try to download a library twice.
trans_undeclared_deps = self._transitive_download_remote_libs(
resolved_remote_libs, all_known_addresses)
undeclared_deps.update(trans_undeclared_deps)
return undeclared_deps
def _transitive_download_remote_libs(self, go_remote_libs, all_known_addresses=None):
"""Recursively attempt to resolve / download all remote transitive deps of go_remote_libs.
Returns a dict<GoRemoteLibrary, set<tuple<str, Address>>>, which maps a go remote library to a
set of unresolved remote dependencies, each dependency expressed as a tuple containing the
the import path of the dependency and the expected target address. If all transitive
dependencies were successfully resolved, returns an empty dict.
Downloads as many invalidated transitive dependencies as possible, and returns as many
undeclared dependencies as possible. However, because the dependencies of a remote library
can only be determined _after_ it has been downloaded, a transitive dependency of an undeclared
remote library will never be detected.
Because go_remote_libraries do not declare dependencies (rather, they are inferred), injects
all successfully resolved transitive dependencies into the build graph.
"""
if not go_remote_libs:
return {}
all_known_addresses = all_known_addresses or set()
all_known_addresses.update(lib.address for lib in go_remote_libs)
resolved_remote_libs = set()
undeclared_deps = defaultdict(set)
go_remote_lib_src = self.context.products.get_data('go_remote_lib_src')
with self.invalidated(go_remote_libs) as invalidation_check:
for vt in invalidation_check.all_vts:
go_remote_lib = vt.target
gopath = vt.results_dir
fetcher = self._get_fetcher(go_remote_lib.import_path)
if not vt.valid:
root = fetcher.root(go_remote_lib.import_path)
fetch_dir = os.path.join(self.workdir, 'fetches')
root_dir = os.path.join(fetch_dir, root)
# Only fetch each remote root once.
if not os.path.exists(root_dir):
with temporary_dir() as tmp_fetch_root:
fetcher.fetch(go_remote_lib.import_path, dest=tmp_fetch_root, rev=go_remote_lib.rev)
safe_mkdir(root_dir)
for path in os.listdir(tmp_fetch_root):
shutil.move(os.path.join(tmp_fetch_root, path), os.path.join(root_dir, path))
# TODO(John Sirois): Circle back and get get rid of this symlink tree.
# GoWorkspaceTask will further symlink a single package from the tree below into a
# target's workspace when it could just be linking from the fetch_dir. The only thing
# standing in the way is a determination of what we want to artifact cache. If we don't
# want to cache fetched zips, linking straight from the fetch_dir works simply. Otherwise
# thought needs to be applied to using the artifact cache directly or synthesizing a
# canonical owner target for the fetched files that 'child' targets (subpackages) can
# depend on and share the fetch from.
dest_dir = os.path.join(gopath, 'src', root)
# We may have been `invalidate`d and not `clean-all`ed so we need a new empty symlink
# chroot to avoid collision; thus `clean=True`.
safe_mkdir(dest_dir, clean=True)
for path in os.listdir(root_dir):
os.symlink(os.path.join(root_dir, path), os.path.join(dest_dir, path))
# Map the fetched remote sources.
pkg = go_remote_lib.import_path
go_remote_lib_src[go_remote_lib] = os.path.join(gopath, 'src', pkg)
for remote_import_path in self._get_remote_import_paths(pkg, gopath=gopath):
fetcher = self._get_fetcher(remote_import_path)
remote_root = fetcher.root(remote_import_path)
spec_path = os.path.join(go_remote_lib.target_base, remote_root)
package_path = GoRemoteLibrary.remote_package_path(remote_root, remote_import_path)
target_name = package_path or os.path.basename(remote_root)
address = Address(spec_path, target_name)
if address not in all_known_addresses:
try:
# If we've already resolved a package from this remote root, its ok to define an
# implicit synthetic remote target for all other packages in the same remote root.
implicit_ok = any(spec_path == a.spec_path for a in all_known_addresses)
remote_lib = self._resolve(go_remote_lib, address, package_path, implicit_ok)
resolved_remote_libs.add(remote_lib)
all_known_addresses.add(address)
except self.UndeclaredRemoteLibError as e:
undeclared_deps[go_remote_lib].add((remote_import_path, e.address))
self.context.build_graph.inject_dependency(go_remote_lib.address, address)
# Recurse after the invalidated block, so the libraries we downloaded are now "valid"
# and thus we don't try to download a library twice.
trans_undeclared_deps = self._transitive_download_remote_libs(resolved_remote_libs,
all_known_addresses)
undeclared_deps.update(trans_undeclared_deps)
return undeclared_deps
