def get_log_text_dict(filename):
"""Return the log contents as a dict"""
logfile_dict = {}
index = 1
for text_line in grablines(filename):
logfile_dict[index] = text_line
index = index + 1
return logfile_dict, index - 1
def __init__(self, options, portdb, event_loop):
self.options = options
self.portdb = portdb
self.event_loop = event_loop
self.added_byte_count = 0
self.added_file_count = 0
self.scheduled_deletion_count = 0
self.delete_count = 0
self.file_owners = {}
self.file_failures = {}
self.start_time = time.time()
self._open_files = []
self.log_success = self._open_log('success', options.success_log, 'a')
self.log_failure = self._open_log('failure', options.failure_log, 'a')
self.distfiles = None
if options.distfiles is not None:
self.distfiles = options.distfiles
self.mirrors = copy.copy(portdb.settings.thirdpartymirrors())
if options.mirror_overrides is not None:
self.mirrors.update(grabdict(options.mirror_overrides))
if options.mirror_skip is not None:
for x in options.mirror_skip.split(","):
self.mirrors[x] = []
self.whitelist = None
if options.whitelist_from is not None:
self.whitelist = set()
for filename in options.whitelist_from:
for line in grablines(filename):
line = line.strip()
if line and not line.startswith("#"):
self.whitelist.add(line)
self.restrict_mirror_exemptions = None
if options.restrict_mirror_exemptions is not None:
self.restrict_mirror_exemptions = frozenset(
options.restrict_mirror_exemptions.split(","))
self.recycle_db = None
if options.recycle_db is not None:
self.recycle_db = self._open_shelve(
options.recycle_db, 'recycle')
self.distfiles_db = None
if options.distfiles_db is not None:
self.distfiles_db = self._open_shelve(
options.distfiles_db, 'distfiles')
self.deletion_db = None
if options.deletion_db is not None:
self.deletion_db = self._open_shelve(
options.deletion_db, 'deletion')
def __init__(self, options, portdb, event_loop):
self.options = options
self.portdb = portdb
self.event_loop = event_loop
self.added_byte_count = 0
self.added_file_count = 0
self.scheduled_deletion_count = 0
self.delete_count = 0
self.file_owners = {}
self.file_failures = {}
self.start_time = time.time()
self._open_files = []
self.log_success = self._open_log('success', options.success_log, 'a')
self.log_failure = self._open_log('failure', options.failure_log, 'a')
self.distfiles = None
if options.distfiles is not None:
self.distfiles = options.distfiles
self.mirrors = copy.copy(portdb.settings.thirdpartymirrors())
if options.mirror_overrides is not None:
self.mirrors.update(grabdict(options.mirror_overrides))
if options.mirror_skip is not None:
for x in options.mirror_skip.split(","):
self.mirrors[x] = []
self.whitelist = None
if options.whitelist_from is not None:
self.whitelist = set()
for filename in options.whitelist_from:
for line in grablines(filename):
line = line.strip()
if line and not line.startswith("#"):
self.whitelist.add(line)
self.restrict_mirror_exemptions = None
if options.restrict_mirror_exemptions is not None:
self.restrict_mirror_exemptions = frozenset(
options.restrict_mirror_exemptions.split(","))
self.recycle_db = None
if options.recycle_db is not None:
self.recycle_db = self._open_shelve(options.recycle_db, 'recycle')
self.distfiles_db = None
if options.distfiles_db is not None:
self.distfiles_db = self._open_shelve(options.distfiles_db,
'distfiles')
self.deletion_db = None
if options.deletion_db is not None:
self.deletion_db = self._open_shelve(options.deletion_db,
'deletion')
def _strip_sig(manifest_path):
"""
Strip an existing signature from a Manifest file.
"""
line_re = ManifestTask._manifest_line_re
lines = grablines(manifest_path)
f = None
try:
f = atomic_ofstream(manifest_path)
for line in lines:
if line_re.match(line) is not None:
f.write(line)
f.close()
f = None
finally:
if f is not None:
f.abort()
def _strip_sig(manifest_path): """ Strip an existing signature from a Manifest file. """ line_re = ManifestTask._manifest_line_re lines = grablines(manifest_path) f = None try: f = atomic_ofstream(manifest_path) for line in lines: if line_re.match(line) is not None: f.write(line) f.close() f = None finally: if f is not None: f.abort()
def getmaskingreason(mycpv,
metadata=None,
settings=None,
portdb=None,
return_location=False,
myrepo=None):
"""
If specified, the myrepo argument is assumed to be valid. This
should be a safe assumption since portdbapi methods always
return valid repo names and valid "repository" metadata from
aux_get.
"""
if settings is None:
settings = portage.settings
if portdb is None:
portdb = portage.portdb
mysplit = catpkgsplit(mycpv)
if not mysplit:
raise ValueError(_("invalid CPV: %s") % mycpv)
if metadata is None:
db_keys = list(portdb._aux_cache_keys)
try:
metadata = dict(
zip(db_keys, portdb.aux_get(mycpv, db_keys, myrepo=myrepo)))
except KeyError:
if not portdb.cpv_exists(mycpv):
raise
else:
if myrepo is None:
myrepo = _gen_valid_repo(metadata["repository"])
elif myrepo is None:
myrepo = metadata.get("repository")
if myrepo is not None:
myrepo = _gen_valid_repo(metadata["repository"])
if metadata is not None and not portage.eapi_is_supported(
metadata["EAPI"]):
# Return early since otherwise we might produce invalid
# results given that the EAPI is not supported. Also,
# metadata is mostly useless in this case since it doesn't
# contain essential things like SLOT.
if return_location:
return (None, None)
return None
# Sometimes we can't access SLOT or repository due to corruption.
pkg = mycpv
try:
pkg.slot
except AttributeError:
pkg = _pkg_str(mycpv, metadata=metadata, repo=myrepo)
cpv_slot_list = [pkg]
mycp = pkg.cp
locations = []
if pkg.repo in settings.repositories:
for repo in settings.repositories[pkg.repo].masters + (
settings.repositories[pkg.repo], ):
locations.append(os.path.join(repo.location, "profiles"))
locations.extend(settings.profiles)
locations.append(
os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH))
locations.reverse()
pmasklists = []
for profile in locations:
pmask_filename = os.path.join(profile, "package.mask")
node = None
for l, recursive_filename in grablines(pmask_filename,
recursive=1,
remember_source_file=True):
if node is None or node[0] != recursive_filename:
node = (recursive_filename, [])
pmasklists.append(node)
node[1].append(l)
pmaskdict = settings._mask_manager._pmaskdict
if mycp in pmaskdict:
for x in pmaskdict[mycp]:
if match_from_list(x, cpv_slot_list):
x = x.without_repo
for pmask in pmasklists:
comment = ""
comment_valid = -1
pmask_filename = pmask[0]
for i in range(len(pmask[1])):
l = pmask[1][i].strip()
try:
l_atom = Atom(l,
allow_repo=True,
allow_wildcard=True).without_repo
except InvalidAtom:
l_atom = None
if l == "":
comment = ""
comment_valid = -1
elif l[0] == "#":
comment += l + "\n"
comment_valid = i + 1
elif l_atom == x:
if comment_valid != i:
comment = ""
if return_location:
return (comment, pmask_filename)
return comment
elif comment_valid != -1:
# Apparently this comment applies to multiple masks, so
# it remains valid until a blank line is encountered.
comment_valid += 1
if return_location:
return (None, None)
return None
def getmaskingreason(mycpv, metadata=None, settings=None, portdb=None, return_location=False, myrepo=None):
"""
If specified, the myrepo argument is assumed to be valid. This
should be a safe assumption since portdbapi methods always
return valid repo names and valid "repository" metadata from
aux_get.
"""
if settings is None:
settings = portage.settings
if portdb is None:
portdb = portage.portdb
mysplit = catpkgsplit(mycpv)
if not mysplit:
raise ValueError(_("invalid CPV: %s") % mycpv)
if metadata is None:
db_keys = list(portdb._aux_cache_keys)
try:
metadata = dict(zip(db_keys, portdb.aux_get(mycpv, db_keys, myrepo=myrepo)))
except KeyError:
if not portdb.cpv_exists(mycpv):
raise
else:
if myrepo is None:
myrepo = _gen_valid_repo(metadata["repository"])
elif myrepo is None:
myrepo = metadata.get("repository")
if myrepo is not None:
myrepo = _gen_valid_repo(metadata["repository"])
if metadata is not None and not portage.eapi_is_supported(metadata["EAPI"]):
# Return early since otherwise we might produce invalid
# results given that the EAPI is not supported. Also,
# metadata is mostly useless in this case since it doesn't
# contain essential things like SLOT.
if return_location:
return (None, None)
else:
return None
# Sometimes we can't access SLOT or repository due to corruption.
pkg = mycpv
if metadata is not None:
pkg = "".join((mycpv, _slot_separator, metadata["SLOT"]))
# At this point myrepo should be None, a valid name, or
# Package.UNKNOWN_REPO which we ignore.
if myrepo is not None and myrepo != Package.UNKNOWN_REPO:
pkg = "".join((pkg, _repo_separator, myrepo))
cpv_slot_list = [pkg]
mycp = mysplit[0] + "/" + mysplit[1]
# XXX- This is a temporary duplicate of code from the config constructor.
locations = [os.path.join(settings["PORTDIR"], "profiles")]
locations.extend(settings.profiles)
for ov in settings["PORTDIR_OVERLAY"].split():
profdir = os.path.join(normalize_path(ov), "profiles")
if os.path.isdir(profdir):
locations.append(profdir)
locations.append(os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH))
locations.reverse()
pmasklists = []
for profile in locations:
pmask_filename = os.path.join(profile, "package.mask")
node = None
for l, recursive_filename in grablines(pmask_filename, recursive=1, remember_source_file=True):
if node is None or node[0] != recursive_filename:
node = (recursive_filename, [])
pmasklists.append(node)
node[1].append(l)
pmaskdict = settings._mask_manager._pmaskdict
if mycp in pmaskdict:
for x in pmaskdict[mycp]:
if match_from_list(x, cpv_slot_list):
x = x.without_repo
for pmask in pmasklists:
comment = ""
comment_valid = -1
pmask_filename = pmask[0]
for i in range(len(pmask[1])):
l = pmask[1][i].strip()
try:
l_atom = Atom(l, allow_repo=True, allow_wildcard=True).without_repo
except InvalidAtom:
l_atom = None
if l == "":
comment = ""
comment_valid = -1
elif l[0] == "#":
comment += l + "\n"
comment_valid = i + 1
elif l_atom == x:
if comment_valid != i:
comment = ""
if return_location:
return (comment, pmask_filename)
else:
return comment
elif comment_valid != -1:
# Apparently this comment applies to multiple masks, so
# it remains valid until a blank line is encountered.
comment_valid += 1
if return_location:
return (None, None)
else:
return None
def getmaskingreason(mycpv,
metadata=None,
settings=None,
portdb=None,
return_location=False,
myrepo=None):
"""
If specified, the myrepo argument is assumed to be valid. This
should be a safe assumption since portdbapi methods always
return valid repo names and valid "repository" metadata from
aux_get.
"""
if settings is None:
settings = portage.settings
if portdb is None:
portdb = portage.portdb
mysplit = catpkgsplit(mycpv)
if not mysplit:
raise ValueError(_("invalid CPV: %s") % mycpv)
if metadata is None:
db_keys = list(portdb._aux_cache_keys)
try:
metadata = dict(
zip(db_keys, portdb.aux_get(mycpv, db_keys, myrepo=myrepo)))
except KeyError:
if not portdb.cpv_exists(mycpv):
raise
else:
if myrepo is None:
myrepo = _gen_valid_repo(metadata["repository"])
elif myrepo is None:
myrepo = metadata.get("repository")
if myrepo is not None:
myrepo = _gen_valid_repo(metadata["repository"])
if metadata is not None and \
not portage.eapi_is_supported(metadata["EAPI"]):
# Return early since otherwise we might produce invalid
# results given that the EAPI is not supported. Also,
# metadata is mostly useless in this case since it doesn't
# contain essential things like SLOT.
if return_location:
return (None, None)
else:
return None
# Sometimes we can't access SLOT or repository due to corruption.
pkg = mycpv
if metadata is not None:
pkg = "".join((mycpv, _slot_separator, metadata["SLOT"]))
# At this point myrepo should be None, a valid name, or
# Package.UNKNOWN_REPO which we ignore.
if myrepo is not None and myrepo != Package.UNKNOWN_REPO:
pkg = "".join((pkg, _repo_separator, myrepo))
cpv_slot_list = [pkg]
mycp = mysplit[0] + "/" + mysplit[1]
# XXX- This is a temporary duplicate of code from the config constructor.
locations = [os.path.join(settings["PORTDIR"], "profiles")]
locations.extend(settings.profiles)
for ov in settings["PORTDIR_OVERLAY"].split():
profdir = os.path.join(normalize_path(ov), "profiles")
if os.path.isdir(profdir):
locations.append(profdir)
locations.append(
os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH))
locations.reverse()
pmasklists = []
for profile in locations:
pmask_filename = os.path.join(profile, "package.mask")
pmasklists.append(
(pmask_filename, grablines(pmask_filename, recursive=1)))
pmaskdict = settings._mask_manager._pmaskdict
if mycp in pmaskdict:
for x in pmaskdict[mycp]:
if match_from_list(x, cpv_slot_list):
x = x.without_repo
for pmask in pmasklists:
comment = ""
comment_valid = -1
pmask_filename = pmask[0]
for i in range(len(pmask[1])):
l = pmask[1][i].strip()
try:
l_atom = Atom(l,
allow_repo=True,
allow_wildcard=True).without_repo
except InvalidAtom:
l_atom = None
if l == "":
comment = ""
comment_valid = -1
elif l[0] == "#":
comment += (l + "\n")
comment_valid = i + 1
elif l_atom == x:
if comment_valid != i:
comment = ""
if return_location:
return (comment, pmask_filename)
else:
return comment
elif comment_valid != -1:
# Apparently this comment applies to multiple masks, so
# it remains valid until a blank line is encountered.
comment_valid += 1
if return_location:
return (None, None)
else:
return None
def getmaskingreason(mycpv, metadata=None, settings=None, portdb=None, return_location=False):
if settings is None:
settings = portage.settings
if portdb is None:
portdb = portage.portdb
mysplit = catpkgsplit(mycpv)
if not mysplit:
raise ValueError(_("invalid CPV: %s") % mycpv)
if metadata is None:
db_keys = list(portdb._aux_cache_keys)
try:
metadata = dict(zip(db_keys, portdb.aux_get(mycpv, db_keys)))
except KeyError:
if not portdb.cpv_exists(mycpv):
raise
if metadata is None:
# Can't access SLOT due to corruption.
cpv_slot_list = [mycpv]
else:
cpv_slot_list = ["%s:%s" % (mycpv, metadata["SLOT"])]
mycp=mysplit[0]+"/"+mysplit[1]
# XXX- This is a temporary duplicate of code from the config constructor.
locations = [os.path.join(settings["PORTDIR"], "profiles")]
locations.extend(settings.profiles)
for ov in settings["PORTDIR_OVERLAY"].split():
profdir = os.path.join(normalize_path(ov), "profiles")
if os.path.isdir(profdir):
locations.append(profdir)
locations.append(os.path.join(settings["PORTAGE_CONFIGROOT"],
USER_CONFIG_PATH))
locations.reverse()
pmasklists = [(x, grablines(os.path.join(x, "package.mask"), recursive=1)) for x in locations]
if mycp in settings.pmaskdict:
for x in settings.pmaskdict[mycp]:
if match_from_list(x, cpv_slot_list):
for pmask in pmasklists:
comment = ""
comment_valid = -1
pmask_filename = os.path.join(pmask[0], "package.mask")
for i in range(len(pmask[1])):
l = pmask[1][i].strip()
if l == "":
comment = ""
comment_valid = -1
elif l[0] == "#":
comment += (l+"\n")
comment_valid = i + 1
elif l == x:
if comment_valid != i:
comment = ""
if return_location:
return (comment, pmask_filename)
else:
return comment
elif comment_valid != -1:
# Apparently this comment applies to muliple masks, so
# it remains valid until a blank line is encountered.
comment_valid += 1
if return_location:
return (None, None)
else:
return None
def __init__(self, options, portdb, event_loop):
self.options = options
self.portdb = portdb
self.event_loop = event_loop
self.added_byte_count = 0
self.added_file_count = 0
self.scheduled_deletion_count = 0
self.delete_count = 0
self.file_owners = {}
self.file_failures = {}
self.start_time = time.time()
self._open_files = []
self.log_success = self._open_log(
"success", getattr(options, "success_log", None), "a"
)
self.log_failure = self._open_log(
"failure", getattr(options, "failure_log", None), "a"
)
self.distfiles = None
if getattr(options, "distfiles", None) is not None:
self.distfiles = options.distfiles
self.mirrors = copy.copy(portdb.settings.thirdpartymirrors())
if getattr(options, "mirror_overrides", None) is not None:
self.mirrors.update(grabdict(options.mirror_overrides))
if getattr(options, "mirror_skip", None) is not None:
for x in options.mirror_skip.split(","):
self.mirrors[x] = []
self.whitelist = None
if getattr(options, "whitelist_from", None) is not None:
self.whitelist = set()
for filename in options.whitelist_from:
for line in grablines(filename):
line = line.strip()
if line and not line.startswith("#"):
self.whitelist.add(line)
self.restrict_mirror_exemptions = None
if getattr(options, "restrict_mirror_exemptions", None) is not None:
self.restrict_mirror_exemptions = frozenset(
options.restrict_mirror_exemptions.split(",")
)
self.recycle_db = None
if getattr(options, "recycle_db", None) is not None:
self.recycle_db = self._open_shelve(options.recycle_db, "recycle")
self.distfiles_db = None
if getattr(options, "distfiles_db", None) is not None:
self.distfiles_db = self._open_shelve(options.distfiles_db, "distfiles")
self.content_db = None
if getattr(options, "content_db", None) is not None:
self.content_db = ContentDB(
self._open_shelve(options.content_db, "content")
)
self.deletion_db = None
if getattr(options, "deletion_db", None) is not None:
self.deletion_db = self._open_shelve(options.deletion_db, "deletion")
self.layout_conf = MirrorLayoutConfig()
if getattr(options, "layout_conf", None) is None:
options.layout_conf = os.path.join(self.distfiles, "layout.conf")
self.layout_conf.read_from_file(options.layout_conf)
self.layouts = self.layout_conf.get_all_layouts()
def getmaskingreason(mycpv, metadata=None, settings=None, portdb=None, return_location=False):
if settings is None:
settings = portage.settings
if portdb is None:
portdb = portage.portdb
mysplit = catpkgsplit(mycpv)
if not mysplit:
raise ValueError(_("invalid CPV: %s") % mycpv)
if metadata is None:
db_keys = list(portdb._aux_cache_keys)
try:
metadata = dict(zip(db_keys, portdb.aux_get(mycpv, db_keys)))
except KeyError:
if not portdb.cpv_exists(mycpv):
raise
if metadata is None:
# Can't access SLOT due to corruption.
cpv_slot_list = [mycpv]
else:
cpv_slot_list = ["%s:%s" % (mycpv, metadata["SLOT"])]
mycp=mysplit[0]+"/"+mysplit[1]
# XXX- This is a temporary duplicate of code from the config constructor.
locations = [os.path.join(settings["PORTDIR"], "profiles")]
locations.extend(settings.profiles)
for ov in settings["PORTDIR_OVERLAY"].split():
profdir = os.path.join(normalize_path(ov), "profiles")
if os.path.isdir(profdir):
locations.append(profdir)
locations.append(os.path.join(settings["PORTAGE_CONFIGROOT"],
USER_CONFIG_PATH))
locations.reverse()
pmasklists = [(x, grablines(os.path.join(x, "package.mask"), recursive=1)) for x in locations]
pmaskdict = settings._mask_manager._pmaskdict
if mycp in pmaskdict:
for x in pmaskdict[mycp]:
if match_from_list(x, cpv_slot_list):
for pmask in pmasklists:
comment = ""
comment_valid = -1
pmask_filename = os.path.join(pmask[0], "package.mask")
for i in range(len(pmask[1])):
l = pmask[1][i].strip()
if l == "":
comment = ""
comment_valid = -1
elif l[0] == "#":
comment += (l+"\n")
comment_valid = i + 1
elif l == x:
if comment_valid != i:
comment = ""
if return_location:
return (comment, pmask_filename)
else:
return comment
elif comment_valid != -1:
# Apparently this comment applies to muliple masks, so
# it remains valid until a blank line is encountered.
comment_valid += 1
if return_location:
return (None, None)
else:
return None