Example #1
0
def UpdateChangeLog(pkgdir, user, msg, skel_path, category, package,
	new=(), removed=(), changed=(), pretend=False, quiet=False):
	"""
	Write an entry to an existing ChangeLog, or create a new one.
	Updates copyright year on changed files, and updates the header of
	ChangeLog with the contents of skel.ChangeLog.
	"""

	if '<root@' in user:
		if not quiet:
			logging.critical('Please set ECHANGELOG_USER or run as non-root')
		return None

	# ChangeLog times are in UTC
	gmtime = time.gmtime()
	year = time.strftime('%Y', gmtime)
	date = time.strftime('%d %b %Y', gmtime)

	# check modified files and the ChangeLog for copyright updates
	# patches and diffs (identified by .patch and .diff) are excluded
	for fn in chain(new, changed):
		if fn.endswith('.diff') or fn.endswith('.patch'):
			continue
		update_copyright(os.path.join(pkgdir, fn), year, pretend=pretend)

	cl_path = os.path.join(pkgdir, 'ChangeLog')
	clold_lines = []
	clnew_lines = []
	old_header_lines = []
	header_lines = []

	clold_file = None
	try:
		clold_file = io.open(_unicode_encode(cl_path,
			encoding=_encodings['fs'], errors='strict'),
			mode='r', encoding=_encodings['repo.content'], errors='replace')
	except EnvironmentError:
		pass

	f, clnew_path = mkstemp()

	# construct correct header first
	try:
		if clold_file is not None:
			# retain header from old ChangeLog
			first_line = True
			for line in clold_file:
				line_strip = line.strip()
				if line_strip and line[:1] != "#":
					clold_lines.append(line)
					break
				# always make sure cat/pkg is up-to-date in case we are
				# moving packages around, or copied from another pkg, or ...
				if first_line:
					if line.startswith('# ChangeLog for'):
						line = '# ChangeLog for %s/%s\n' % (category, package)
					first_line = False
				old_header_lines.append(line)
				header_lines.append(_update_copyright_year(year, line))
				if not line_strip:
					break

		clskel_file = None
		if not header_lines:
			# delay opening this until we find we need a header
			try:
				clskel_file = io.open(_unicode_encode(skel_path,
					encoding=_encodings['fs'], errors='strict'),
					mode='r', encoding=_encodings['repo.content'],
					errors='replace')
			except EnvironmentError:
				pass

		if clskel_file is not None:
			# read skel.ChangeLog up to first empty line
			for line in clskel_file:
				line_strip = line.strip()
				if not line_strip:
					break
				line = line.replace('<CATEGORY>', category)
				line = line.replace('<PACKAGE_NAME>', package)
				line = _update_copyright_year(year, line)
				header_lines.append(line)
			header_lines.append('\n')
			clskel_file.close()

		# write new ChangeLog entry
		clnew_lines.extend(header_lines)
		newebuild = False
		for fn in new:
			if not fn.endswith('.ebuild'):
				continue
			ebuild = fn.split(os.sep)[-1][0:-7] 
			clnew_lines.append('*%s (%s)\n' % (ebuild, date))
			newebuild = True
		if newebuild:
			clnew_lines.append('\n')
		trivial_files = ('ChangeLog', 'Manifest')
		display_new = ['+' + elem for elem in new
			if elem not in trivial_files]
		display_removed = ['-' + elem for elem in removed]
		display_changed = [elem for elem in changed
			if elem not in trivial_files]
		if not (display_new or display_removed or display_changed):
			# If there's nothing else to display, show one of the
			# trivial files.
			for fn in trivial_files:
				if fn in new:
					display_new = ['+' + fn]
					break
				elif fn in changed:
					display_changed = [fn]
					break

		display_new.sort()
		display_removed.sort()
		display_changed.sort()

		mesg = '%s; %s %s:' % (date, user, ', '.join(chain(
			display_new, display_removed, display_changed)))
		for line in textwrap.wrap(mesg, 80, \
				initial_indent='  ', subsequent_indent='  ', \
				break_on_hyphens=False):
			clnew_lines.append('%s\n' % line)
		for line in textwrap.wrap(msg, 80, \
				initial_indent='  ', subsequent_indent='  '):
			clnew_lines.append('%s\n' % line)
		# Don't append a trailing newline if the file is new.
		if clold_file is not None:
			clnew_lines.append('\n')

		f = io.open(f, mode='w', encoding=_encodings['repo.content'],
			errors='backslashreplace')

		for line in clnew_lines:
			f.write(line)

		# append stuff from old ChangeLog
		if clold_file is not None:

			if clold_lines:
				# clold_lines may contain a saved non-header line
				# that we want to write first.
				# Also, append this line to clnew_lines so that the
				# unified_diff call doesn't show it as removed.
				for line in clold_lines:
					f.write(line)
					clnew_lines.append(line)

			else:
				# ensure that there is no more than one blank
				# line after our new entry
				for line in clold_file:
					if line.strip():
						f.write(line)
						break

			# Now prepend old_header_lines to clold_lines, for use
			# in the unified_diff call below.
			clold_lines = old_header_lines + clold_lines

			# Trim any trailing newlines.
			lines = clold_file.readlines()
			clold_file.close()
			while lines and lines[-1] == '\n':
				del lines[-1]
			f.writelines(lines)
		f.close()

		# show diff
		if not quiet:
			for line in difflib.unified_diff(clold_lines, clnew_lines,
					fromfile=cl_path, tofile=cl_path, n=0):
				util.writemsg_stdout(line, noiselevel=-1)
			util.writemsg_stdout("\n", noiselevel=-1)

		if pretend:
			# remove what we've done
			os.remove(clnew_path)
		else:
			# rename to ChangeLog, and set permissions
			try:
				clold_stat = os.stat(cl_path)
			except OSError:
				clold_stat = None

			shutil.move(clnew_path, cl_path)

			if clold_stat is None:
				util.apply_permissions(cl_path, mode=0o644)
			else:
				util.apply_stat_permissions(cl_path, clold_stat)

		if clold_file is None:
			return True
		else:
			return False
	except IOError as e:
		err = 'Repoman is unable to create/write to Changelog.new file: %s' % (e,)
		logging.critical(err)
		# try to remove if possible
		try:
			os.remove(clnew_path)
		except OSError:
			pass
		return None
Example #2
0
def update_copyright(fn_path, year, pretend=False):
	"""
	Check file for a Copyright statement, and update its year.  The
	patterns used for replacing copyrights are taken from echangelog.
	Only the first lines of each file that start with a hash ('#') are
	considered, until a line is found that doesn't start with a hash.
	Files are read and written in binary mode, so that this function
	will work correctly with files encoded in any character set, as
	long as the copyright statements consist of plain ASCII.
	"""

	try:
		fn_hdl = io.open(_unicode_encode(
			fn_path, encoding=_encodings['fs'], errors='strict'),
			mode='rb')
	except EnvironmentError:
		return

	orig_header = []
	new_header = []

	for line in fn_hdl:
		line_strip = line.strip()
		orig_header.append(line)
		if not line_strip or line_strip[:1] != b'#':
			new_header.append(line)
			break

		line = update_copyright_year(year, line)
		new_header.append(line)

	difflines = 0
	for diffline in difflib.unified_diff(
		[_unicode_decode(diffline) for diffline in orig_header],
		[_unicode_decode(diffline) for diffline in new_header],
		fromfile=fn_path, tofile=fn_path, n=0):
		util.writemsg_stdout(diffline, noiselevel=-1)
		difflines += 1
	util.writemsg_stdout("\n", noiselevel=-1)

	# unified diff has three lines to start with
	if difflines > 3 and not pretend:
		# write new file with changed header
		f, fnnew_path = mkstemp()
		f = io.open(f, mode='wb')
		for line in new_header:
			f.write(line)
		for line in fn_hdl:
			f.write(line)
		f.close()
		try:
			fn_stat = os.stat(fn_path)
		except OSError:
			fn_stat = None

		shutil.move(fnnew_path, fn_path)

		if fn_stat is None:
			util.apply_permissions(fn_path, mode=0o644)
		else:
			util.apply_stat_permissions(fn_path, fn_stat)
	fn_hdl.close()
Example #3
0
def UpdateChangeLog(
	pkgdir, user, msg, skel_path, category, package,
	new=(), removed=(), changed=(), pretend=False, quiet=False):
	"""
	Write an entry to an existing ChangeLog, or create a new one.
	Updates copyright year on changed files, and updates the header of
	ChangeLog with the contents of skel.ChangeLog.
	"""

	if '<root@' in user:
		if not quiet:
			logging.critical('Please set ECHANGELOG_USER or run as non-root')
		return None

	# ChangeLog times are in UTC
	gmtime = time.gmtime()
	year = time.strftime('%Y', gmtime)
	date = time.strftime('%d %b %Y', gmtime)

	# check modified files and the ChangeLog for copyright updates
	# patches and diffs (identified by .patch and .diff) are excluded
	for fn in chain(new, changed):
		if fn.endswith('.diff') or fn.endswith('.patch'):
			continue
		update_copyright(os.path.join(pkgdir, fn), year, pretend=pretend)

	cl_path = os.path.join(pkgdir, 'ChangeLog')
	clold_lines = []
	clnew_lines = []
	old_header_lines = []
	header_lines = []

	clold_file = None
	try:
		clold_file = io.open(_unicode_encode(
			cl_path, encoding=_encodings['fs'], errors='strict'),
			mode='r', encoding=_encodings['repo.content'], errors='replace')
	except EnvironmentError:
		pass

	f, clnew_path = mkstemp()

	# construct correct header first
	try:
		if clold_file is not None:
			# retain header from old ChangeLog
			first_line = True
			for line in clold_file:
				line_strip = line.strip()
				if line_strip and line[:1] != "#":
					clold_lines.append(line)
					break
				# always make sure cat/pkg is up-to-date in case we are
				# moving packages around, or copied from another pkg, or ...
				if first_line:
					if line.startswith('# ChangeLog for'):
						line = '# ChangeLog for %s/%s\n' % (category, package)
					first_line = False
				old_header_lines.append(line)
				header_lines.append(update_copyright_year(year, line))
				if not line_strip:
					break

		clskel_file = None
		if not header_lines:
			# delay opening this until we find we need a header
			try:
				clskel_file = io.open(_unicode_encode(
					skel_path, encoding=_encodings['fs'], errors='strict'),
					mode='r', encoding=_encodings['repo.content'],
					errors='replace')
			except EnvironmentError:
				pass

		if clskel_file is not None:
			# read skel.ChangeLog up to first empty line
			for line in clskel_file:
				line_strip = line.strip()
				if not line_strip:
					break
				line = line.replace('<CATEGORY>', category)
				line = line.replace('<PACKAGE_NAME>', package)
				line = update_copyright_year(year, line)
				header_lines.append(line)
			header_lines.append('\n')
			clskel_file.close()

		# write new ChangeLog entry
		clnew_lines.extend(header_lines)
		newebuild = False
		for fn in new:
			if not fn.endswith('.ebuild'):
				continue
			ebuild = fn.split(os.sep)[-1][0:-7]
			clnew_lines.append('*%s (%s)\n' % (ebuild, date))
			newebuild = True
		if newebuild:
			clnew_lines.append('\n')
		trivial_files = ('ChangeLog', 'Manifest')
		display_new = [
			'+' + elem
			for elem in new
			if elem not in trivial_files]
		display_removed = [
			'-' + elem
			for elem in removed]
		display_changed = [
			elem for elem in changed
			if elem not in trivial_files]
		if not (display_new or display_removed or display_changed):
			# If there's nothing else to display, show one of the
			# trivial files.
			for fn in trivial_files:
				if fn in new:
					display_new = ['+' + fn]
					break
				elif fn in changed:
					display_changed = [fn]
					break

		display_new.sort()
		display_removed.sort()
		display_changed.sort()

		mesg = '%s; %s %s:' % (date, user, ', '.join(chain(
			display_new, display_removed, display_changed)))
		for line in textwrap.wrap(
			mesg, 80, initial_indent='  ', subsequent_indent='  ',
			break_on_hyphens=False):
			clnew_lines.append('%s\n' % line)
		for line in textwrap.wrap(
			msg, 80, initial_indent='  ', subsequent_indent='  '):
			clnew_lines.append('%s\n' % line)
		# Don't append a trailing newline if the file is new.
		if clold_file is not None:
			clnew_lines.append('\n')

		f = io.open(
			f, mode='w', encoding=_encodings['repo.content'],
			errors='backslashreplace')

		for line in clnew_lines:
			f.write(line)

		# append stuff from old ChangeLog
		if clold_file is not None:

			if clold_lines:
				# clold_lines may contain a saved non-header line
				# that we want to write first.
				# Also, append this line to clnew_lines so that the
				# unified_diff call doesn't show it as removed.
				for line in clold_lines:
					f.write(line)
					clnew_lines.append(line)

			else:
				# ensure that there is no more than one blank
				# line after our new entry
				for line in clold_file:
					if line.strip():
						f.write(line)
						break

			# Now prepend old_header_lines to clold_lines, for use
			# in the unified_diff call below.
			clold_lines = old_header_lines + clold_lines

			# Trim any trailing newlines.
			lines = clold_file.readlines()
			clold_file.close()
			while lines and lines[-1] == '\n':
				del lines[-1]
			f.writelines(lines)
		f.close()

		# show diff
		if not quiet:
			for line in difflib.unified_diff(
				clold_lines, clnew_lines,
				fromfile=cl_path, tofile=cl_path, n=0):
				util.writemsg_stdout(line, noiselevel=-1)
			util.writemsg_stdout("\n", noiselevel=-1)

		if pretend:
			# remove what we've done
			os.remove(clnew_path)
		else:
			# rename to ChangeLog, and set permissions
			try:
				clold_stat = os.stat(cl_path)
			except OSError:
				clold_stat = None

			shutil.move(clnew_path, cl_path)

			if clold_stat is None:
				util.apply_permissions(cl_path, mode=0o644)
			else:
				util.apply_stat_permissions(cl_path, clold_stat)

		if clold_file is None:
			return True
		else:
			return False
	except IOError as e:
		err = 'Repoman is unable to create/write to Changelog.new file: %s' % (e,)
		logging.critical(err)
		# try to remove if possible
		try:
			os.remove(clnew_path)
		except OSError:
			pass
		return None
Example #4
0
def fetch_metadata_xsd(metadata_xsd, repoman_settings):
	"""
	Fetch metadata.xsd if it doesn't exist or the ctime is older than
	metadata_xsd_ctime_interval.
	@rtype: bool
	@return: True if successful, otherwise False
	"""

	must_fetch = True
	metadata_xsd_st = None
	current_time = int(time.time())
	try:
		metadata_xsd_st = os.stat(metadata_xsd)
	except EnvironmentError as e:
		if e.errno not in (errno.ENOENT, errno.ESTALE):
			raise
		del e
	else:
		# Trigger fetch if metadata.xsd mtime is old or clock is wrong.
		if abs(current_time - metadata_xsd_st.st_ctime) \
			< metadata_xsd_ctime_interval:
			must_fetch = False

	if must_fetch:
		print()
		print(
			"%s the local copy of metadata.xsd "
			"needs to be refetched, doing that now" % green("***"))
		print()
		parsed_url = urlparse(metadata_xsd_uri)
		setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
		fcmd = repoman_settings.get(setting)
		if not fcmd:
			fcmd = repoman_settings.get('FETCHCOMMAND')
			if not fcmd:
				logging.error("FETCHCOMMAND is unset")
				return False

		destdir = repoman_settings["DISTDIR"]
		fd, metadata_xsd_tmp = tempfile.mkstemp(
			prefix='metadata.xsd.', dir=destdir)
		os.close(fd)

		try:
			if not portage.getbinpkg.file_get(
				metadata_xsd_uri, destdir, fcmd=fcmd,
				filename=os.path.basename(metadata_xsd_tmp)):
				logging.error(
					"failed to fetch metadata.xsd from '%s'" % metadata_xsd_uri)
				return False

			try:
				portage.util.apply_secpass_permissions(
					metadata_xsd_tmp,
					gid=portage.data.portage_gid, mode=0o664, mask=0o2)
			except portage.exception.PortageException:
				pass

			shutil.move(metadata_xsd_tmp, metadata_xsd)
		finally:
			try:
				os.unlink(metadata_xsd_tmp)
			except OSError:
				pass

	return True
Example #5
0
def update_copyright(fn_path, year, pretend=False):
    """
	Check file for a Copyright statement, and update its year.  The
	patterns used for replacing copyrights are taken from echangelog.
	Only the first lines of each file that start with a hash ('#') are
	considered, until a line is found that doesn't start with a hash.
	Files are read and written in binary mode, so that this function
	will work correctly with files encoded in any character set, as
	long as the copyright statements consist of plain ASCII.
	"""

    try:
        fn_hdl = io.open(_unicode_encode(fn_path,
                                         encoding=_encodings['fs'],
                                         errors='strict'),
                         mode='rb')
    except EnvironmentError:
        return

    orig_header = []
    new_header = []

    for line in fn_hdl:
        line_strip = line.strip()
        orig_header.append(line)
        if not line_strip or line_strip[:1] != b'#':
            new_header.append(line)
            break

        line = _update_copyright_year(year, line)
        new_header.append(line)

    difflines = 0
    for line in difflib.unified_diff(
        [_unicode_decode(line) for line in orig_header],
        [_unicode_decode(line) for line in new_header],
            fromfile=fn_path,
            tofile=fn_path,
            n=0):
        util.writemsg_stdout(line, noiselevel=-1)
        difflines += 1
    util.writemsg_stdout("\n", noiselevel=-1)

    # unified diff has three lines to start with
    if difflines > 3 and not pretend:
        # write new file with changed header
        f, fnnew_path = mkstemp()
        f = io.open(f, mode='wb')
        for line in new_header:
            f.write(line)
        for line in fn_hdl:
            f.write(line)
        f.close()
        try:
            fn_stat = os.stat(fn_path)
        except OSError:
            fn_stat = None

        shutil.move(fnnew_path, fn_path)

        if fn_stat is None:
            util.apply_permissions(fn_path, mode=0o644)
        else:
            util.apply_stat_permissions(fn_path, fn_stat)
    fn_hdl.close()
Example #6
0
def fetch_metadata_xsd(metadata_xsd, repoman_settings):
	"""
	Fetch metadata.xsd if it doesn't exist or the ctime is older than
	metadata_xsd_ctime_interval.
	@rtype: bool
	@return: True if successful, otherwise False
	"""

	must_fetch = True
	metadata_xsd_st = None
	current_time = int(time.time())
	try:
		metadata_xsd_st = os.stat(metadata_xsd)
	except EnvironmentError as e:
		if e.errno not in (errno.ENOENT, errno.ESTALE):
			raise
		del e
	else:
		# Trigger fetch if metadata.xsd mtime is old or clock is wrong.
		if abs(current_time - metadata_xsd_st.st_ctime) \
			< metadata_xsd_ctime_interval:
			must_fetch = False

	if must_fetch:
		print()
		print(
			"%s the local copy of metadata.xsd "
			"needs to be refetched, doing that now" % green("***"))
		print()
		parsed_url = urlparse(metadata_xsd_uri)
		setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
		fcmd = repoman_settings.get(setting)
		if not fcmd:
			fcmd = repoman_settings.get('FETCHCOMMAND')
			if not fcmd:
				logging.error("FETCHCOMMAND is unset")
				return False

		destdir = repoman_settings["DISTDIR"]
		fd, metadata_xsd_tmp = tempfile.mkstemp(
			prefix='metadata.xsd.', dir=destdir)
		os.close(fd)

		try:
			if not portage.getbinpkg.file_get(
				metadata_xsd_uri, destdir, fcmd=fcmd,
				filename=os.path.basename(metadata_xsd_tmp)):
				logging.error(
					"failed to fetch metadata.xsd from '%s'" % metadata_xsd_uri)
				return False

			try:
				portage.util.apply_secpass_permissions(
					metadata_xsd_tmp,
					gid=portage.data.portage_gid, mode=0o664, mask=0o2)
			except portage.exception.PortageException:
				pass

			shutil.move(metadata_xsd_tmp, metadata_xsd)
		finally:
			try:
				os.unlink(metadata_xsd_tmp)
			except OSError:
				pass

	return True
Example #7
0
def update_copyright(fn_path,
                     year,
                     pretend=False,
                     owner=None,
                     update_owner=False,
                     add_copyright=False):
    """
	Check file for a Copyright statement, and update its year.  The
	patterns used for replacing copyrights are taken from echangelog.
	Only the first lines of each file that start with a hash ('#') are
	considered, until a line is found that doesn't start with a hash.
	Files are read and written in binary mode, so that this function
	will work correctly with files encoded in any character set, as
	long as the copyright statements consist of plain ASCII.

	@param fn_path: file path
	@type str
	@param year: current year
	@type str
	@param pretend: pretend mode
	@type bool
	@rtype: bool
	@return: True if copyright update was needed, False otherwise
	"""

    try:
        fn_hdl = io.open(_unicode_encode(fn_path,
                                         encoding=_encodings['fs'],
                                         errors='strict'),
                         mode='rb')
    except EnvironmentError:
        return

    owner = _unicode_encode(owner) or b'Gentoo Authors'

    orig_header = []
    new_header = []
    has_copyright = False

    for line in fn_hdl:
        line_strip = line.strip()
        orig_header.append(line)
        if not line_strip or line_strip[:1] != b'#':
            new_header.append(line)
            break
        has_copyright = max(has_copyright, line.startswith(b'# Copyright '))
        # update date range
        line = update_copyright_year(year, line)
        # now check for and add COPYRIGHT_OWNER
        if update_owner and owner not in line:
            line = add_owner(owner, line)
        new_header.append(line)
    if not has_copyright and add_copyright:
        new_copyright = b' '.join(
            [b'# Copyright', _unicode_encode(year), owner]) + b'\n'
        new_header.insert(0, new_copyright)

    difflines = 0
    for diffline in difflib.unified_diff(
        [_unicode_decode(diffline) for diffline in orig_header],
        [_unicode_decode(diffline) for diffline in new_header],
            fromfile=fn_path,
            tofile=fn_path,
            n=0):
        util.writemsg_stdout(diffline, noiselevel=-1)
        difflines += 1
    util.writemsg_stdout("\n", noiselevel=-1)

    # unified diff has three lines to start with
    if difflines > 3 and not pretend:
        # write new file with changed header
        f, fnnew_path = mkstemp()
        f = io.open(f, mode='wb')
        for line in new_header:
            f.write(line)
        for line in fn_hdl:
            f.write(line)
        f.close()
        try:
            fn_stat = os.stat(fn_path)
        except OSError:
            fn_stat = None

        shutil.move(fnnew_path, fn_path)

        if fn_stat is None:
            util.apply_permissions(fn_path, mode=0o644)
        else:
            util.apply_stat_permissions(fn_path, fn_stat)
    fn_hdl.close()
    return difflines > 3