def UpdateChangeLog(pkgdir, user, msg, skel_path, category, package,
new=(), removed=(), changed=(), pretend=False, quiet=False):
"""
Write an entry to an existing ChangeLog, or create a new one.
Updates copyright year on changed files, and updates the header of
ChangeLog with the contents of skel.ChangeLog.
"""
if '<root@' in user:
if not quiet:
logging.critical('Please set ECHANGELOG_USER or run as non-root')
return None
# ChangeLog times are in UTC
gmtime = time.gmtime()
year = time.strftime('%Y', gmtime)
date = time.strftime('%d %b %Y', gmtime)
# check modified files and the ChangeLog for copyright updates
# patches and diffs (identified by .patch and .diff) are excluded
for fn in chain(new, changed):
if fn.endswith('.diff') or fn.endswith('.patch'):
continue
update_copyright(os.path.join(pkgdir, fn), year, pretend=pretend)
cl_path = os.path.join(pkgdir, 'ChangeLog')
clold_lines = []
clnew_lines = []
old_header_lines = []
header_lines = []
clold_file = None
try:
clold_file = io.open(_unicode_encode(cl_path,
encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['repo.content'], errors='replace')
except EnvironmentError:
pass
f, clnew_path = mkstemp()
# construct correct header first
try:
if clold_file is not None:
# retain header from old ChangeLog
first_line = True
for line in clold_file:
line_strip = line.strip()
if line_strip and line[:1] != "#":
clold_lines.append(line)
break
# always make sure cat/pkg is up-to-date in case we are
# moving packages around, or copied from another pkg, or ...
if first_line:
if line.startswith('# ChangeLog for'):
line = '# ChangeLog for %s/%s\n' % (category, package)
first_line = False
old_header_lines.append(line)
header_lines.append(_update_copyright_year(year, line))
if not line_strip:
break
clskel_file = None
if not header_lines:
# delay opening this until we find we need a header
try:
clskel_file = io.open(_unicode_encode(skel_path,
encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['repo.content'],
errors='replace')
except EnvironmentError:
pass
if clskel_file is not None:
# read skel.ChangeLog up to first empty line
for line in clskel_file:
line_strip = line.strip()
if not line_strip:
break
line = line.replace('<CATEGORY>', category)
line = line.replace('<PACKAGE_NAME>', package)
line = _update_copyright_year(year, line)
header_lines.append(line)
header_lines.append('\n')
clskel_file.close()
# write new ChangeLog entry
clnew_lines.extend(header_lines)
newebuild = False
for fn in new:
if not fn.endswith('.ebuild'):
continue
ebuild = fn.split(os.sep)[-1][0:-7]
clnew_lines.append('*%s (%s)\n' % (ebuild, date))
newebuild = True
if newebuild:
clnew_lines.append('\n')
trivial_files = ('ChangeLog', 'Manifest')
display_new = ['+' + elem for elem in new
if elem not in trivial_files]
display_removed = ['-' + elem for elem in removed]
display_changed = [elem for elem in changed
if elem not in trivial_files]
if not (display_new or display_removed or display_changed):
# If there's nothing else to display, show one of the
# trivial files.
for fn in trivial_files:
if fn in new:
display_new = ['+' + fn]
break
elif fn in changed:
display_changed = [fn]
break
display_new.sort()
display_removed.sort()
display_changed.sort()
mesg = '%s; %s %s:' % (date, user, ', '.join(chain(
display_new, display_removed, display_changed)))
for line in textwrap.wrap(mesg, 80, \
initial_indent=' ', subsequent_indent=' ', \
break_on_hyphens=False):
clnew_lines.append('%s\n' % line)
for line in textwrap.wrap(msg, 80, \
initial_indent=' ', subsequent_indent=' '):
clnew_lines.append('%s\n' % line)
# Don't append a trailing newline if the file is new.
if clold_file is not None:
clnew_lines.append('\n')
f = io.open(f, mode='w', encoding=_encodings['repo.content'],
errors='backslashreplace')
for line in clnew_lines:
f.write(line)
# append stuff from old ChangeLog
if clold_file is not None:
if clold_lines:
# clold_lines may contain a saved non-header line
# that we want to write first.
# Also, append this line to clnew_lines so that the
# unified_diff call doesn't show it as removed.
for line in clold_lines:
f.write(line)
clnew_lines.append(line)
else:
# ensure that there is no more than one blank
# line after our new entry
for line in clold_file:
if line.strip():
f.write(line)
break
# Now prepend old_header_lines to clold_lines, for use
# in the unified_diff call below.
clold_lines = old_header_lines + clold_lines
# Trim any trailing newlines.
lines = clold_file.readlines()
clold_file.close()
while lines and lines[-1] == '\n':
del lines[-1]
f.writelines(lines)
f.close()
# show diff
if not quiet:
for line in difflib.unified_diff(clold_lines, clnew_lines,
fromfile=cl_path, tofile=cl_path, n=0):
util.writemsg_stdout(line, noiselevel=-1)
util.writemsg_stdout("\n", noiselevel=-1)
if pretend:
# remove what we've done
os.remove(clnew_path)
else:
# rename to ChangeLog, and set permissions
try:
clold_stat = os.stat(cl_path)
except OSError:
clold_stat = None
shutil.move(clnew_path, cl_path)
if clold_stat is None:
util.apply_permissions(cl_path, mode=0o644)
else:
util.apply_stat_permissions(cl_path, clold_stat)
if clold_file is None:
return True
else:
return False
except IOError as e:
err = 'Repoman is unable to create/write to Changelog.new file: %s' % (e,)
logging.critical(err)
# try to remove if possible
try:
os.remove(clnew_path)
except OSError:
pass
return None
def update_copyright(fn_path, year, pretend=False):
"""
Check file for a Copyright statement, and update its year. The
patterns used for replacing copyrights are taken from echangelog.
Only the first lines of each file that start with a hash ('#') are
considered, until a line is found that doesn't start with a hash.
Files are read and written in binary mode, so that this function
will work correctly with files encoded in any character set, as
long as the copyright statements consist of plain ASCII.
"""
try:
fn_hdl = io.open(_unicode_encode(
fn_path, encoding=_encodings['fs'], errors='strict'),
mode='rb')
except EnvironmentError:
return
orig_header = []
new_header = []
for line in fn_hdl:
line_strip = line.strip()
orig_header.append(line)
if not line_strip or line_strip[:1] != b'#':
new_header.append(line)
break
line = update_copyright_year(year, line)
new_header.append(line)
difflines = 0
for diffline in difflib.unified_diff(
[_unicode_decode(diffline) for diffline in orig_header],
[_unicode_decode(diffline) for diffline in new_header],
fromfile=fn_path, tofile=fn_path, n=0):
util.writemsg_stdout(diffline, noiselevel=-1)
difflines += 1
util.writemsg_stdout("\n", noiselevel=-1)
# unified diff has three lines to start with
if difflines > 3 and not pretend:
# write new file with changed header
f, fnnew_path = mkstemp()
f = io.open(f, mode='wb')
for line in new_header:
f.write(line)
for line in fn_hdl:
f.write(line)
f.close()
try:
fn_stat = os.stat(fn_path)
except OSError:
fn_stat = None
shutil.move(fnnew_path, fn_path)
if fn_stat is None:
util.apply_permissions(fn_path, mode=0o644)
else:
util.apply_stat_permissions(fn_path, fn_stat)
fn_hdl.close()
def UpdateChangeLog(
pkgdir, user, msg, skel_path, category, package,
new=(), removed=(), changed=(), pretend=False, quiet=False):
"""
Write an entry to an existing ChangeLog, or create a new one.
Updates copyright year on changed files, and updates the header of
ChangeLog with the contents of skel.ChangeLog.
"""
if '<root@' in user:
if not quiet:
logging.critical('Please set ECHANGELOG_USER or run as non-root')
return None
# ChangeLog times are in UTC
gmtime = time.gmtime()
year = time.strftime('%Y', gmtime)
date = time.strftime('%d %b %Y', gmtime)
# check modified files and the ChangeLog for copyright updates
# patches and diffs (identified by .patch and .diff) are excluded
for fn in chain(new, changed):
if fn.endswith('.diff') or fn.endswith('.patch'):
continue
update_copyright(os.path.join(pkgdir, fn), year, pretend=pretend)
cl_path = os.path.join(pkgdir, 'ChangeLog')
clold_lines = []
clnew_lines = []
old_header_lines = []
header_lines = []
clold_file = None
try:
clold_file = io.open(_unicode_encode(
cl_path, encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['repo.content'], errors='replace')
except EnvironmentError:
pass
f, clnew_path = mkstemp()
# construct correct header first
try:
if clold_file is not None:
# retain header from old ChangeLog
first_line = True
for line in clold_file:
line_strip = line.strip()
if line_strip and line[:1] != "#":
clold_lines.append(line)
break
# always make sure cat/pkg is up-to-date in case we are
# moving packages around, or copied from another pkg, or ...
if first_line:
if line.startswith('# ChangeLog for'):
line = '# ChangeLog for %s/%s\n' % (category, package)
first_line = False
old_header_lines.append(line)
header_lines.append(update_copyright_year(year, line))
if not line_strip:
break
clskel_file = None
if not header_lines:
# delay opening this until we find we need a header
try:
clskel_file = io.open(_unicode_encode(
skel_path, encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['repo.content'],
errors='replace')
except EnvironmentError:
pass
if clskel_file is not None:
# read skel.ChangeLog up to first empty line
for line in clskel_file:
line_strip = line.strip()
if not line_strip:
break
line = line.replace('<CATEGORY>', category)
line = line.replace('<PACKAGE_NAME>', package)
line = update_copyright_year(year, line)
header_lines.append(line)
header_lines.append('\n')
clskel_file.close()
# write new ChangeLog entry
clnew_lines.extend(header_lines)
newebuild = False
for fn in new:
if not fn.endswith('.ebuild'):
continue
ebuild = fn.split(os.sep)[-1][0:-7]
clnew_lines.append('*%s (%s)\n' % (ebuild, date))
newebuild = True
if newebuild:
clnew_lines.append('\n')
trivial_files = ('ChangeLog', 'Manifest')
display_new = [
'+' + elem
for elem in new
if elem not in trivial_files]
display_removed = [
'-' + elem
for elem in removed]
display_changed = [
elem for elem in changed
if elem not in trivial_files]
if not (display_new or display_removed or display_changed):
# If there's nothing else to display, show one of the
# trivial files.
for fn in trivial_files:
if fn in new:
display_new = ['+' + fn]
break
elif fn in changed:
display_changed = [fn]
break
display_new.sort()
display_removed.sort()
display_changed.sort()
mesg = '%s; %s %s:' % (date, user, ', '.join(chain(
display_new, display_removed, display_changed)))
for line in textwrap.wrap(
mesg, 80, initial_indent=' ', subsequent_indent=' ',
break_on_hyphens=False):
clnew_lines.append('%s\n' % line)
for line in textwrap.wrap(
msg, 80, initial_indent=' ', subsequent_indent=' '):
clnew_lines.append('%s\n' % line)
# Don't append a trailing newline if the file is new.
if clold_file is not None:
clnew_lines.append('\n')
f = io.open(
f, mode='w', encoding=_encodings['repo.content'],
errors='backslashreplace')
for line in clnew_lines:
f.write(line)
# append stuff from old ChangeLog
if clold_file is not None:
if clold_lines:
# clold_lines may contain a saved non-header line
# that we want to write first.
# Also, append this line to clnew_lines so that the
# unified_diff call doesn't show it as removed.
for line in clold_lines:
f.write(line)
clnew_lines.append(line)
else:
# ensure that there is no more than one blank
# line after our new entry
for line in clold_file:
if line.strip():
f.write(line)
break
# Now prepend old_header_lines to clold_lines, for use
# in the unified_diff call below.
clold_lines = old_header_lines + clold_lines
# Trim any trailing newlines.
lines = clold_file.readlines()
clold_file.close()
while lines and lines[-1] == '\n':
del lines[-1]
f.writelines(lines)
f.close()
# show diff
if not quiet:
for line in difflib.unified_diff(
clold_lines, clnew_lines,
fromfile=cl_path, tofile=cl_path, n=0):
util.writemsg_stdout(line, noiselevel=-1)
util.writemsg_stdout("\n", noiselevel=-1)
if pretend:
# remove what we've done
os.remove(clnew_path)
else:
# rename to ChangeLog, and set permissions
try:
clold_stat = os.stat(cl_path)
except OSError:
clold_stat = None
shutil.move(clnew_path, cl_path)
if clold_stat is None:
util.apply_permissions(cl_path, mode=0o644)
else:
util.apply_stat_permissions(cl_path, clold_stat)
if clold_file is None:
return True
else:
return False
except IOError as e:
err = 'Repoman is unable to create/write to Changelog.new file: %s' % (e,)
logging.critical(err)
# try to remove if possible
try:
os.remove(clnew_path)
except OSError:
pass
return None
def fetch_metadata_xsd(metadata_xsd, repoman_settings):
"""
Fetch metadata.xsd if it doesn't exist or the ctime is older than
metadata_xsd_ctime_interval.
@rtype: bool
@return: True if successful, otherwise False
"""
must_fetch = True
metadata_xsd_st = None
current_time = int(time.time())
try:
metadata_xsd_st = os.stat(metadata_xsd)
except EnvironmentError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
else:
# Trigger fetch if metadata.xsd mtime is old or clock is wrong.
if abs(current_time - metadata_xsd_st.st_ctime) \
< metadata_xsd_ctime_interval:
must_fetch = False
if must_fetch:
print()
print(
"%s the local copy of metadata.xsd "
"needs to be refetched, doing that now" % green("***"))
print()
parsed_url = urlparse(metadata_xsd_uri)
setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
fcmd = repoman_settings.get(setting)
if not fcmd:
fcmd = repoman_settings.get('FETCHCOMMAND')
if not fcmd:
logging.error("FETCHCOMMAND is unset")
return False
destdir = repoman_settings["DISTDIR"]
fd, metadata_xsd_tmp = tempfile.mkstemp(
prefix='metadata.xsd.', dir=destdir)
os.close(fd)
try:
if not portage.getbinpkg.file_get(
metadata_xsd_uri, destdir, fcmd=fcmd,
filename=os.path.basename(metadata_xsd_tmp)):
logging.error(
"failed to fetch metadata.xsd from '%s'" % metadata_xsd_uri)
return False
try:
portage.util.apply_secpass_permissions(
metadata_xsd_tmp,
gid=portage.data.portage_gid, mode=0o664, mask=0o2)
except portage.exception.PortageException:
pass
shutil.move(metadata_xsd_tmp, metadata_xsd)
finally:
try:
os.unlink(metadata_xsd_tmp)
except OSError:
pass
return True
def update_copyright(fn_path, year, pretend=False):
"""
Check file for a Copyright statement, and update its year. The
patterns used for replacing copyrights are taken from echangelog.
Only the first lines of each file that start with a hash ('#') are
considered, until a line is found that doesn't start with a hash.
Files are read and written in binary mode, so that this function
will work correctly with files encoded in any character set, as
long as the copyright statements consist of plain ASCII.
"""
try:
fn_hdl = io.open(_unicode_encode(fn_path,
encoding=_encodings['fs'],
errors='strict'),
mode='rb')
except EnvironmentError:
return
orig_header = []
new_header = []
for line in fn_hdl:
line_strip = line.strip()
orig_header.append(line)
if not line_strip or line_strip[:1] != b'#':
new_header.append(line)
break
line = _update_copyright_year(year, line)
new_header.append(line)
difflines = 0
for line in difflib.unified_diff(
[_unicode_decode(line) for line in orig_header],
[_unicode_decode(line) for line in new_header],
fromfile=fn_path,
tofile=fn_path,
n=0):
util.writemsg_stdout(line, noiselevel=-1)
difflines += 1
util.writemsg_stdout("\n", noiselevel=-1)
# unified diff has three lines to start with
if difflines > 3 and not pretend:
# write new file with changed header
f, fnnew_path = mkstemp()
f = io.open(f, mode='wb')
for line in new_header:
f.write(line)
for line in fn_hdl:
f.write(line)
f.close()
try:
fn_stat = os.stat(fn_path)
except OSError:
fn_stat = None
shutil.move(fnnew_path, fn_path)
if fn_stat is None:
util.apply_permissions(fn_path, mode=0o644)
else:
util.apply_stat_permissions(fn_path, fn_stat)
fn_hdl.close()
def fetch_metadata_xsd(metadata_xsd, repoman_settings):
"""
Fetch metadata.xsd if it doesn't exist or the ctime is older than
metadata_xsd_ctime_interval.
@rtype: bool
@return: True if successful, otherwise False
"""
must_fetch = True
metadata_xsd_st = None
current_time = int(time.time())
try:
metadata_xsd_st = os.stat(metadata_xsd)
except EnvironmentError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
else:
# Trigger fetch if metadata.xsd mtime is old or clock is wrong.
if abs(current_time - metadata_xsd_st.st_ctime) \
< metadata_xsd_ctime_interval:
must_fetch = False
if must_fetch:
print()
print(
"%s the local copy of metadata.xsd "
"needs to be refetched, doing that now" % green("***"))
print()
parsed_url = urlparse(metadata_xsd_uri)
setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
fcmd = repoman_settings.get(setting)
if not fcmd:
fcmd = repoman_settings.get('FETCHCOMMAND')
if not fcmd:
logging.error("FETCHCOMMAND is unset")
return False
destdir = repoman_settings["DISTDIR"]
fd, metadata_xsd_tmp = tempfile.mkstemp(
prefix='metadata.xsd.', dir=destdir)
os.close(fd)
try:
if not portage.getbinpkg.file_get(
metadata_xsd_uri, destdir, fcmd=fcmd,
filename=os.path.basename(metadata_xsd_tmp)):
logging.error(
"failed to fetch metadata.xsd from '%s'" % metadata_xsd_uri)
return False
try:
portage.util.apply_secpass_permissions(
metadata_xsd_tmp,
gid=portage.data.portage_gid, mode=0o664, mask=0o2)
except portage.exception.PortageException:
pass
shutil.move(metadata_xsd_tmp, metadata_xsd)
finally:
try:
os.unlink(metadata_xsd_tmp)
except OSError:
pass
return True
def update_copyright(fn_path,
year,
pretend=False,
owner=None,
update_owner=False,
add_copyright=False):
"""
Check file for a Copyright statement, and update its year. The
patterns used for replacing copyrights are taken from echangelog.
Only the first lines of each file that start with a hash ('#') are
considered, until a line is found that doesn't start with a hash.
Files are read and written in binary mode, so that this function
will work correctly with files encoded in any character set, as
long as the copyright statements consist of plain ASCII.
@param fn_path: file path
@type str
@param year: current year
@type str
@param pretend: pretend mode
@type bool
@rtype: bool
@return: True if copyright update was needed, False otherwise
"""
try:
fn_hdl = io.open(_unicode_encode(fn_path,
encoding=_encodings['fs'],
errors='strict'),
mode='rb')
except EnvironmentError:
return
owner = _unicode_encode(owner) or b'Gentoo Authors'
orig_header = []
new_header = []
has_copyright = False
for line in fn_hdl:
line_strip = line.strip()
orig_header.append(line)
if not line_strip or line_strip[:1] != b'#':
new_header.append(line)
break
has_copyright = max(has_copyright, line.startswith(b'# Copyright '))
# update date range
line = update_copyright_year(year, line)
# now check for and add COPYRIGHT_OWNER
if update_owner and owner not in line:
line = add_owner(owner, line)
new_header.append(line)
if not has_copyright and add_copyright:
new_copyright = b' '.join(
[b'# Copyright', _unicode_encode(year), owner]) + b'\n'
new_header.insert(0, new_copyright)
difflines = 0
for diffline in difflib.unified_diff(
[_unicode_decode(diffline) for diffline in orig_header],
[_unicode_decode(diffline) for diffline in new_header],
fromfile=fn_path,
tofile=fn_path,
n=0):
util.writemsg_stdout(diffline, noiselevel=-1)
difflines += 1
util.writemsg_stdout("\n", noiselevel=-1)
# unified diff has three lines to start with
if difflines > 3 and not pretend:
# write new file with changed header
f, fnnew_path = mkstemp()
f = io.open(f, mode='wb')
for line in new_header:
f.write(line)
for line in fn_hdl:
f.write(line)
f.close()
try:
fn_stat = os.stat(fn_path)
except OSError:
fn_stat = None
shutil.move(fnnew_path, fn_path)
if fn_stat is None:
util.apply_permissions(fn_path, mode=0o644)
else:
util.apply_stat_permissions(fn_path, fn_stat)
fn_hdl.close()
return difflines > 3