def create_file_repo(self, repo_props=EmptyDict, create_repo=False):
if self.transport.publish_cache_contains(self.publisher):
return
if create_repo:
try:
# For compatibility reasons, assume that
# repositories created using pkgsend
# should be in version 3 format (single
# publisher only).
sr.repository_create(self.path, version=3)
except sr.RepositoryExistsError:
# Already exists, nothing to do.
pass
except (apx.ApiException, sr.RepositoryError) as e:
raise TransactionOperationError(None, msg=str(e))
try:
repo = sr.Repository(properties=repo_props, root=self.path)
except EnvironmentError as e:
raise TransactionOperationError(
None,
msg=_("An error occurred while trying to "
"initialize the repository directory "
"structures:\n{0}").format(e))
except cfg.ConfigError as e:
raise TransactionRepositoryConfigError(str(e))
except sr.RepositoryInvalidError as e:
raise TransactionRepositoryInvalidError(str(e))
except sr.RepositoryError as e:
raise TransactionOperationError(None, msg=str(e))
self.transport.publish_cache_repository(self.publisher, repo)
def get_repo(self, auto_create=False):
if auto_create:
try:
sr.repository_create(self.__dir)
except sr.RepositoryExistsError:
# Already exists, nothing to do.
pass
return sr.Repository(cfgpathname=self.__cfg_file,
root=self.__dir, writable_root=self.__writable_root)
def get_repo(uri):
parts = urlparse.urlparse(uri, "file", allow_fragments=0)
path = urllib.url2pathname(parts[2])
try:
return repo.Repository(root=path)
except cfg.ConfigError as e:
raise repo.RepositoryError(_("The specified "
"repository's configuration data is not "
"valid:\n{0}").format(e))
def create_file_repo(self, origin_url, repo_props=EmptyDict,
create_repo=False, refresh_index=True):
if self.transport.publish_cache_contains(self.publisher):
return
try:
repo = sr.Repository(auto_create=create_repo,
properties=repo_props, repo_root=self.path,
refresh_index=refresh_index)
except EnvironmentError, e:
raise TransactionOperationError(None, msg=_(
"An error occurred while trying to "
"initialize the repository directory "
"structures:\n%s") % e)
class FileTransaction(object):
"""Provides a publishing interface for file-based repositories."""
# Used to avoid the overhead of initializing the repository for
# successive transactions.
__repo_cache = {}
def __init__(self, origin_url, create_repo=False, pkg_name=None,
trans_id=None):
scheme, netloc, path, params, query, fragment = \
urlparse.urlparse(origin_url, "file", allow_fragments=0)
path = urllib.url2pathname(path)
repo_cache = self.__class__.__repo_cache
if not os.path.isabs(path):
raise TransactionRepositoryURLError(origin_url,
msg=_("Not an absolute path."))
if origin_url not in repo_cache:
scfg = config.SvrConfig(path, None, None,
auto_create=create_repo)
try:
scfg.init_dirs()
except (config.SvrConfigError, EnvironmentError), e:
raise TransactionOperationError(None, msg=_(
"An error occurred while trying to "
"initialize the repository directory "
"structures:\n%s") % e)
scfg.acquire_in_flight()
try:
scfg.acquire_catalog()
except catalog.CatalogPermissionsException, e:
raise TransactionOperationError(None,
origin_url, msg=str(e))
try:
repo_cache[origin_url] = repo.Repository(scfg)
except rc.InvalidAttributeValueError, e:
raise TransactionOperationError(None,
msg=_("The specified repository's "
"configuration data is not "
"valid:\n%s") % e)
def _get_publishers(root):
"""Given a repository root, return the list of available publishers,
along with the default publisher/prefix."""
try:
# we don't set writable_root, as we don't want to take the hit
# on potentially building an index here.
repository = sr.Repository(root=root, read_only=True)
if repository.version != 4:
raise DepotException(
_("pkg.depot-config only supports v4 repositories"))
except Exception as e:
raise DepotException(e)
all_pubs = [pub.prefix for pub in repository.get_publishers()]
try:
default_pub = repository.cfg.get_property("publisher", "prefix")
except cfg.UnknownPropertyError:
default_pub = None
return all_pubs, default_pub, repository.get_status()
def main_func():
global temp_root, repo_modified, repo_finished, repo_uri, tracker
global dry_run
global_settings.client_name = PKG_CLIENT_NAME
try:
opts, pargs = getopt.getopt(sys.argv[1:], "?c:i:np:r:s:u", ["help"])
except getopt.GetoptError as e:
usage(_("illegal option -- {0}").format(e.opt))
dry_run = False
ref_repo_uri = None
repo_uri = os.getenv("PKG_REPO", None)
changes = set()
ignores = set()
publishers = set()
cmp_policy = CMP_ALL
processed_pubs = 0
for opt, arg in opts:
if opt == "-c":
changes.add(arg)
elif opt == "-i":
ignores.add(arg)
elif opt == "-n":
dry_run = True
elif opt == "-p":
publishers.add(arg)
elif opt == "-r":
ref_repo_uri = misc.parse_uri(arg)
elif opt == "-s":
repo_uri = misc.parse_uri(arg)
elif opt == "-u":
cmp_policy = CMP_UNSIGNED
elif opt == "-?" or opt == "--help":
usage(retcode=pkgdefs.EXIT_OK)
if pargs:
usage(_("Unexpected argument(s): {0}").format(" ".join(pargs)))
if not repo_uri:
usage(_("A target repository must be provided."))
if not ref_repo_uri:
usage(_("A reference repository must be provided."))
target = publisher.RepositoryURI(misc.parse_uri(repo_uri))
if target.scheme != "file":
abort(err=_("Target repository must be filesystem-based."))
try:
target_repo = sr.Repository(read_only=dry_run,
root=target.get_pathname())
except sr.RepositoryError as e:
abort(str(e))
# Use the tmp directory in target repo for efficient file rename since
# files are in the same file system.
temp_root = target_repo.temp_root
if not os.path.exists(temp_root):
os.makedirs(temp_root)
ref_incoming_dir = tempfile.mkdtemp(dir=temp_root)
ref_pkg_root = tempfile.mkdtemp(dir=temp_root)
ref_xport, ref_xport_cfg = transport.setup_transport()
ref_xport_cfg.incoming_root = ref_incoming_dir
ref_xport_cfg.pkg_root = ref_pkg_root
transport.setup_publisher(ref_repo_uri,
"ref",
ref_xport,
ref_xport_cfg,
remote_prefix=True)
ref_repo = None
ref = publisher.RepositoryURI(misc.parse_uri(ref_repo_uri))
if ref.scheme == "file":
try:
ref_repo = sr.Repository(read_only=dry_run,
root=ref.get_pathname())
except sr.RepositoryError as e:
abort(str(e))
tracker = get_tracker()
for pub in target_repo.publishers:
if publishers and pub not in publishers \
and '*' not in publishers:
continue
msg(_("Processing packages for publisher {0} ...").format(pub))
# Find the matching pub in the ref repo.
for ref_pub in ref_xport_cfg.gen_publishers():
if ref_pub.prefix == pub:
found = True
break
else:
txt = _("Publisher {0} not found in reference "
"repository.").format(pub)
if publishers:
abort(err=txt)
else:
txt += _(" Skipping.")
msg(txt)
continue
processed_pubs += 1
rev = do_reversion(pub, ref_pub, target_repo, ref_xport, changes,
ignores, cmp_policy, ref_repo, ref, ref_xport_cfg)
# Only rebuild catalog if anything got actually reversioned.
if rev and not dry_run:
msg(_("Rebuilding repository catalog."))
target_repo.rebuild(pub=pub)
repo_finished = True
ret = pkgdefs.EXIT_OK
if processed_pubs == 0:
msg(_("No matching publishers could be found."))
ret = pkgdefs.EXIT_OOPS
cleanup()
return ret
def setup(self, request):
"""Builds a dictionary of sr.Repository objects, keyed by the
repository prefix, and ensures our search indexes exist."""
def get_repo_paths():
"""Return a dictionary of repository paths, and writable
root directories for the repositories we're serving."""
repo_paths = {}
for key in request.wsgi_environ:
if key.startswith("PKG5_REPOSITORY"):
prefix = key.replace("PKG5_REPOSITORY_", "")
repo_paths[prefix] = \
request.wsgi_environ[key]
writable_root = \
request.wsgi_environ.get(
"PKG5_WRITABLE_ROOT_{0}".format(prefix))
repo_paths[prefix] = \
(request.wsgi_environ[key],
writable_root)
return repo_paths
if repositories:
return
# if running the pkg5 test suite, store the correct proto area
pkg5_test_proto = request.wsgi_environ.get("PKG5_TEST_PROTO", "")
repository_lock.acquire()
repo_paths = get_repo_paths()
# We must ensure our BackgroundTask object has at least as many
# slots available as we have repositories, to allow the indexes
# to be refreshed. Ideally, we'd also account for a slot
# per-publisher, per-repository, but that might be overkill on a
# system with many repositories and many publishers that rarely
# get 'pkgrepo refresh' requests.
self.bgtask = BackgroundTask(
len(repo_paths),
busy_url="{0}/depot/depot-keepalive".format(request.base))
self.bgtask.start()
for prefix in repo_paths:
path, writable_root = repo_paths[prefix]
try:
repo = sr.Repository(root=path,
read_only=True,
writable_root=writable_root)
except sr.RepositoryError as e:
print("Unable to load repository: {0}".format(e))
continue
repositories[prefix] = repo
dconf = sd.DepotConfig()
if writable_root is not None:
self.bgtask.put(repo.refresh_index)
depot = DepotBUI(repo,
dconf,
writable_root,
pkg5_test_proto=pkg5_test_proto)
depot_buis[prefix] = depot
repository_lock.release()
try:
sr.repository_create(inst_root, properties=repo_props)
except sr.RepositoryExistsError:
# Already exists, nothing to do.
pass
except (api_errors.ApiException, sr.RepositoryError) as _e:
emsg("pkg.depotd: {0}".format(_e))
sys.exit(1)
try:
sort_file_max_size = dconf.get_property("pkg",
"sort_file_max_size")
repo = sr.Repository(cfgpathname=repo_config_file,
log_obj=cherrypy, mirror=mirror, properties=repo_props,
read_only=readonly, root=inst_root,
sort_file_max_size=sort_file_max_size,
writable_root=writable_root)
except (RuntimeError, sr.RepositoryError) as _e:
emsg("pkg.depotd: {0}".format(_e))
sys.exit(1)
except search_errors.IndexingException as _e:
emsg("pkg.depotd: {0}".format(str(_e)), "INDEX")
sys.exit(1)
except api_errors.ApiException as _e:
emsg("pkg.depotd: {0}".format(str(_e)))
sys.exit(1)
if not rebuild and not add_content and not repo.mirror and \
not (repo.read_only and not repo.writable_root):
# Automatically update search indexes on startup if not already
def get_repo(self, auto_create=False):
return sr.Repository(auto_create=auto_create,
cfgpathname=self.__cfg_file,
repo_root=self.__dir)
try:
scfg.init_dirs()
except (config.SvrConfigError, EnvironmentError), e:
raise repo.RepositoryError(_("An error occurred while "
"trying to initialize the repository directory "
"structures:\n%s") % e)
scfg.acquire_in_flight()
try:
scfg.acquire_catalog()
except catalog.CatalogPermissionsException, e:
raise repo.RepositoryError(str(e))
try:
repo_cache[uri] = repo.Repository(scfg)
except rc.InvalidAttributeValueError, e:
raise repo.RepositoryError(_("The specified repository's "
"configuration data is not valid:\n%s") % e)
return repo_cache[uri]
def fetch_manifest(src_uri, pfmri):
"""Return the manifest data for package-fmri 'fmri' from the repository
at 'src_uri'."""
if src_uri.startswith("file://"):
try:
r = get_repo(src_uri)
m = file(r.manifest(pfmri), "rb")
except (EnvironmentError, repo.RepositoryError), e: