class Repo:
gh: Optional[GitHub]
def __init__(self, config: configparser.ConfigParser):
self.myaddress = config.get('lilac', 'email')
self.mymaster = config.get('lilac', 'master')
self.repomail = config.get('repository', 'email')
self.name = config.get('repository', 'name')
self.trim_ansi_codes = not config.getboolean(
'smtp', 'use_ansi', fallback=False)
self.repodir = Path(config.get('repository', 'repodir')).expanduser()
self.ms = MailService(config)
github_token = config.get('lilac', 'github_token', fallback=None)
if github_token:
self.gh = GitHub(config.get('lilac', 'github_token',
fallback=None))
else:
self.gh = None
self.mods: LilacMods = {
} # to be filled by self.load_all_lilac_and_report()
@lru_cache()
def maintainer_from_github(self, username: str) -> Optional[Maintainer]:
if self.gh is None:
raise ValueError('未设置 github token，无法从 GitHub 取得用户 Email 地址')
userinfo = self.gh.get_user_info(username)
if userinfo['email']:
return Maintainer(userinfo['name'], userinfo['email'], username)
else:
return None
@lru_cache()
def find_maintainers(self, mod: LilacMod) -> List[Maintainer]:
ret = []
errors = []
maintainers: List[Dict[str, str]] = getattr(mod, 'maintainers', None)
if maintainers is not None:
for m in maintainers:
if 'github' in m and 'email' in m:
ret.append(
Maintainer.from_email_address(m['email'], m['github']))
elif 'github' in m:
try:
u = self.maintainer_from_github(m['github'])
except Exception as e:
errors.append(f'从 GitHub 获取用户 Email 地址时出错：{e!r}')
else:
if u is None:
errors.append(
f'GitHub 用户 {m["github"]} 未公开 Email 地址')
else:
ret.append(u)
else:
logger.error('unsupported maintainer info: %r', m)
errors.append(f'不支持的格式：{m!r}')
continue
if not ret or errors:
# fallback to git
dir = self.repodir / mod.pkgbase
git_maintainer = self.find_maintainer_by_git(dir)
if errors:
error_str = '\n'.join(errors)
self.sendmail(
git_maintainer,
subject=f'{mod.pkgbase} 的 maintainers 信息有误',
msg=f"以下 maintainers 信息有误，请修正。\n\n{error_str}\n",
)
if not ret:
logger.warning(
"lilac doesn't give out maintainers for %s, "
"fallback to git.", mod.pkgbase)
return [git_maintainer]
else:
return ret
def find_maintainer_by_git(
self,
dir: Path = Path('.'),
file: str = '*',
) -> Maintainer:
me = self.myaddress
cmd = [
"git",
"log",
"--format=%H %an <%ae>",
"--",
file,
]
p = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
universal_newlines=True,
cwd=dir,
)
try:
while True:
line = p.stdout.readline()
commit, author = line.rstrip().split(None, 1)
if me not in author:
return Maintainer.from_email_address(author)
finally:
p.terminate()
def report_error(self, subject: str, msg: str) -> None:
self.ms.sendmail(self.mymaster, subject, msg)
def send_error_report(
self,
mod: Union[LilacMod, str],
*,
msg: Optional[str] = None,
exc: Optional[Tuple[Exception, str]] = None,
subject: Optional[str] = None,
) -> None:
if msg is None and exc is None:
raise TypeError('send_error_report received inefficient args')
if isinstance(mod, str):
maintainers = [self.find_maintainer_by_git(file=mod)]
pkgbase = mod
else:
maintainers = self.find_maintainers(mod)
pkgbase = mod.pkgbase
msgs = []
if msg is not None:
msgs.append(msg)
if exc is not None:
exception, tb = exc
if isinstance(exception, subprocess.CalledProcessError):
subject_real = subject or '在编译软件包 %s 时发生错误'
msgs.append('命令执行失败！\n\n命令 %r 返回了错误号 %d。' \
'命令的输出如下：\n\n%s' % (
exception.cmd, exception.returncode, exception.output))
msgs.append('调用栈如下：\n\n' + tb)
elif isinstance(exception, api.AurDownloadError):
subject_real = subject or '在获取AUR包 %s 时发生错误'
msgs.append('获取AUR包失败！\n\n')
msgs.append('调用栈如下：\n\n' + tb)
else:
subject_real = subject or '在编译软件包 %s 时发生未知错误'
msgs.append('发生未知错误！调用栈如下：\n\n' + tb)
else:
if subject is None:
raise ValueError('subject should be given but not')
subject_real = subject
if '%s' in subject_real:
subject_real = subject_real % pkgbase
if build_output:
msgs.append('编译命令输出如下：\n\n' + build_output)
msg = '\n'.join(msgs)
if self.trim_ansi_codes:
msg = ansi_escape_re.sub('', msg)
addresses = [str(x) for x in maintainers]
logger.debug('mail to %s:\nsubject: %s\nbody: %s', addresses,
subject_real, msg[:200])
self.sendmail(addresses, subject_real, msg)
def sendmail(self, who: Union[str, List[str], Maintainer], subject: str,
msg: str) -> None:
if isinstance(who, Maintainer):
who = str(who)
self.ms.sendmail(who, subject, msg)
def send_repo_mail(self, subject: str, msg: str) -> None:
self.ms.sendmail(self.repomail, subject, msg)
def manages(self, dep) -> bool:
return dep.pkgdir.name in self.mods
def load_all_lilac_and_report(self) -> Set[str]:
self.mods, errors = lilacpy.load_all(self.repodir)
failed = set(errors)
for name, exc_info in errors.items():
tb_lines = traceback.format_exception(*exc_info)
tb = ''.join(tb_lines)
logger.error('error while loading lilac.py for %s',
name,
exc_info=exc_info)
exc = exc_info[1]
if not isinstance(exc, Exception):
raise
self.send_error_report(name,
exc=(exc, tb),
subject='为软件包 %s 载入 lilac.py 时失败')
build_logger_old.error('%s failed', name)
build_logger.exception('lilac.py error', pkgbase=name)
return failed
class Repo:
def __init__(self, config):
self.myaddress = config.get('lilac', 'email')
self.mymaster = config.get('lilac', 'master')
self.repomail = config.get('repository', 'email')
self.name = config.get('repository', 'name')
self.trim_ansi_codes = not config.getboolean(
'smtp', 'use_ansi', fallback=False)
self.repodir = Path(config.get('repository', 'repodir')).expanduser()
self.ms = MailService(config)
github_token = config.get('lilac', 'github_token', fallback=None)
if github_token:
self.gh = GitHub(config.get('lilac', 'github_token', fallback=None))
else:
self.gh = None
self.mods: LilacMods = {} # to be filled by self.load_all_lilac_and_report()
@lru_cache()
def maintainer_from_github(self, username: str) -> Optional[Maintainer]:
if self.gh is None:
raise ValueError('未设置 github token，无法从 GitHub 取得用户 Email 地址')
userinfo = self.gh.get_user_info(username)
if userinfo['email']:
return Maintainer(userinfo['name'], userinfo['email'], username)
else:
return None
@lru_cache()
def find_maintainers(self, mod: LilacMod) -> List[Maintainer]:
ret = []
errors = []
maintainers: List[Dict[str, str]] = getattr(mod, 'maintainers', None)
if maintainers is not None:
for m in maintainers:
if 'github' in m and 'email' in m:
ret.append(
Maintainer.from_email_address(m['email'], m['github'])
)
elif 'github' in m:
try:
u = self.maintainer_from_github(m['github'])
except Exception as e:
errors.append(f'从 GitHub 获取用户 Email 地址时出错：{e!r}')
else:
if u is None:
errors.append(f'GitHub 用户 {m["github"]} 未公开 Email 地址')
else:
ret.append(u)
else:
logger.error('unsupported maintainer info: %r', m)
errors.append(f'不支持的格式：{m!r}')
continue
if not ret or errors:
# fallback to git
dir = self.repodir / mod.pkgbase
git_maintainer = self.find_maintainer_by_git(dir)
if errors:
error_str = '\n'.join(errors)
self.sendmail(
git_maintainer,
subject = f'{mod.pkgbase} 的 maintainers 信息有误',
msg = f"以下 maintainers 信息有误，请修正。\n\n{error_str}\n",
)
if not ret:
logger.warning("lilac doesn't give out maintainers for %s, "
"fallback to git.", mod.pkgbase)
return [git_maintainer]
else:
return ret
def find_maintainer_by_git(
self,
dir: Path = Path('.'),
file: str = '*',
) -> Maintainer:
me = self.myaddress
cmd = [
"git", "log", "--format=%H %an <%ae>", "--", file,
]
p = subprocess.Popen(
cmd, stdout=subprocess.PIPE, universal_newlines=True,
cwd = dir,
)
try:
while True:
line = p.stdout.readline()
commit, author = line.rstrip().split(None, 1)
if me not in author:
return Maintainer.from_email_address(author)
finally:
p.terminate()
def report_error(self, subject: str, msg: str) -> None:
self.ms.sendmail(self.mymaster, subject, msg)
def send_error_report(
self,
mod: Union[LilacMod, str], *,
msg: Optional[str] = None,
exc: Optional[Tuple[Exception, str]] = None,
subject: Optional[str] = None,
) -> None:
if msg is None and exc is None:
raise TypeError('send_error_report received inefficient args')
if isinstance(mod, str):
maintainers = [self.find_maintainer_by_git(file=mod)]
pkgbase = mod
else:
maintainers = self.find_maintainers(mod)
pkgbase = mod.pkgbase
msgs = []
if msg is not None:
msgs.append(msg)
if exc is not None:
exception, tb = exc
if isinstance(exception, subprocess.CalledProcessError):
subject_real = subject or '在编译软件包 %s 时发生错误'
msgs.append('命令执行失败！\n\n命令 %r 返回了错误号 %d。' \
'命令的输出如下：\n\n%s' % (
exception.cmd, exception.returncode, exception.output))
msgs.append('调用栈如下：\n\n' + tb)
elif isinstance(exception, api.AurDownloadError):
subject_real = subject or '在获取AUR包 %s 时发生错误'
msgs.append('获取AUR包失败！\n\n')
msgs.append('调用栈如下：\n\n' + tb)
else:
subject_real = subject or '在编译软件包 %s 时发生未知错误'
msgs.append('发生未知错误！调用栈如下：\n\n' + tb)
else:
if subject is None:
raise ValueError('subject should be given but not')
subject_real = subject
if '%s' in subject_real:
subject_real = subject_real % pkgbase
if build_output:
msgs.append('编译命令输出如下：\n\n' + build_output)
msg = '\n'.join(msgs)
if self.trim_ansi_codes:
msg = ansi_escape_re.sub('', msg)
addresses = [str(x) for x in maintainers]
logger.debug('mail to %s:\nsubject: %s\nbody: %s',
addresses, subject_real, msg[:200])
self.sendmail(addresses, subject_real, msg)
def sendmail(self, who: Union[str, List[str], Maintainer],
subject: str, msg: str) -> None:
if isinstance(who, Maintainer):
who = str(who)
self.ms.sendmail(who, subject, msg)
def send_repo_mail(self, subject: str, msg: str) -> None:
self.ms.sendmail(self.repomail, subject, msg)
def manages(self, dep) -> bool:
return dep.pkgdir.name in self.mods
def load_all_lilac_and_report(self) -> Set[str]:
self.mods, errors = lilacpy.load_all(self.repodir)
failed = set(errors)
for name, exc_info in errors.items():
tb_lines = traceback.format_exception(*exc_info)
tb = ''.join(tb_lines)
logger.error('error while loading lilac.py for %s', name, exc_info=exc_info)
exc = exc_info[1]
if not isinstance(exc, Exception):
raise
self.send_error_report(name, exc=(exc, tb),
subject='为软件包 %s 载入 lilac.py 时失败')
build_logger_old.error('%s failed', name)
build_logger.exception('lilac.py error', pkgbase = name)
return failed
class Repo:
gh: Optional[GitHub]
def __init__(self, config: Dict[str, Any]) -> None:
self.myaddress = config['lilac']['email']
self.mymaster = config['lilac']['master']
self.logurl_template = config['lilac'].get('logurl')
self.repomail = config['repository']['email']
self.name = config['repository']['name']
self.trim_ansi_codes = not config['smtp']['use_ansi']
self.repodir = Path(config['repository']['repodir']).expanduser()
self.ms = MailService(config)
github_token = config['lilac'].get('github_token')
if github_token:
self.gh = GitHub(github_token)
else:
self.gh = None
self.mods: LilacMods = {
} # to be filled by self.load_all_lilac_and_report()
@lru_cache()
def maintainer_from_github(self, username: str) -> Optional[Maintainer]:
if self.gh is None:
raise ValueError('未设置 github token，无法从 GitHub 取得用户 Email 地址')
userinfo = self.gh.get_user_info(username)
if userinfo['email']:
return Maintainer(userinfo['name'] or username, userinfo['email'],
username)
else:
return None
def parse_maintainers(
self,
ms: List[Dict[str, str]],
) -> Tuple[List[Maintainer], List[str]]:
ret = []
errors = []
for m in ms:
if 'github' in m and 'email' in m:
ret.append(
Maintainer.from_email_address(m['email'], m['github']))
elif 'github' in m:
try:
u = self.maintainer_from_github(m['github'])
except Exception as e:
errors.append(f'从 GitHub 获取用户 Email 地址时出错：{e!r}')
else:
if u is None:
errors.append(f'GitHub 用户 {m["github"]} 未公开 Email 地址')
else:
ret.append(u)
else:
logger.error('unsupported maintainer info: %r', m)
errors.append(f'不支持的格式：{m!r}')
continue
return ret, errors
def find_dependents(
self,
pkgbase: str,
) -> List[LilacMod]:
ret = []
for mod in self.mods.values():
ds = getattr(mod, 'repo_depends', ())
if any(x == pkgbase for x, y in ds):
ret.append(mod)
return ret
@lru_cache()
def find_maintainers(
self,
mod: LilacMod,
fallback_git: bool = True,
) -> List[Maintainer]:
ret: List[Maintainer] = []
errors: List[str] = []
maintainers: List[Dict[str, str]] = getattr(mod, 'maintainers', None)
if maintainers is not None:
if maintainers:
ret, errors = self.parse_maintainers(maintainers)
else:
dependents = self.find_dependents(mod.pkgbase)
for dmod in dependents:
dmaints = self.find_maintainers(dmod, fallback_git=False)
ret.extend(dmaints)
if (not ret and fallback_git) or errors:
# fallback to git
dir = self.repodir / mod.pkgbase
git_maintainer = self.find_maintainer_by_git(dir)
if errors:
error_str = '\n'.join(errors)
self.sendmail(
git_maintainer,
subject=f'{mod.pkgbase} 的 maintainers 信息有误',
msg=f"以下 maintainers 信息有误，请修正。\n\n{error_str}\n",
)
if not ret and fallback_git:
logger.warning(
"lilac doesn't give out maintainers for %s, "
"fallback to git.", mod.pkgbase)
return [git_maintainer]
else:
return ret
def find_maintainer_by_git(
self,
dir: Path = Path('.'),
file: str = '*',
) -> Maintainer:
me = self.myaddress
cmd = [
"git",
"log",
"--format=%H %an <%ae>",
"--",
file,
]
p = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
universal_newlines=True,
cwd=dir,
)
try:
stdout = p.stdout
assert stdout
while True:
line = stdout.readline()
commit, author = line.rstrip().split(None, 1)
if me not in author:
return Maintainer.from_email_address(author)
finally:
p.terminate()
def report_error(self, subject: str, msg: str) -> None:
self.ms.sendmail(self.mymaster, subject, msg)
def send_error_report(
self,
mod: Union[LilacMod, str],
*,
msg: Optional[str] = None,
exc: Optional[Tuple[Exception, str]] = None,
subject: Optional[str] = None,
logfile: Optional[Path] = None,
) -> None:
if msg is None and exc is None:
raise TypeError('send_error_report received insufficient args')
if isinstance(mod, str):
maintainers = [self.find_maintainer_by_git(file=mod)]
pkgbase = mod
else:
maintainers = self.find_maintainers(mod)
pkgbase = mod.pkgbase
msgs = []
if msg is not None:
msgs.append(msg)
if exc is not None:
exception, tb = exc
if isinstance(exception, subprocess.CalledProcessError):
subject_real = subject or '在打包软件包 %s 时发生错误'
msgs.append('命令执行失败！\n\n命令 %r 返回了错误号 %d。' %
(exception.cmd, exception.returncode))
if exception.output:
msgs.append('命令的输出如下：\n\n%s' % exception.output)
msgs.append('调用栈如下：\n\n' + tb)
elif isinstance(exception, api.AurDownloadError):
subject_real = subject or '在获取AUR包 %s 时发生错误'
msgs.append('获取AUR包失败！\n\n')
msgs.append('调用栈如下：\n\n' + tb)
elif isinstance(exception, TimeoutError):
subject_real = subject or '打包软件包 %s 超时'
else:
subject_real = subject or '在打包软件包 %s 时发生未知错误'
msgs.append('发生未知错误！调用栈如下：\n\n' + tb)
else:
if subject is None:
raise ValueError('subject should be given but not')
subject_real = subject
if '%s' in subject_real:
subject_real = subject_real % pkgbase
if logfile:
try:
with logfile.open(errors='surrogateescape') as f:
build_output = f.read()
if build_output:
log_header = '打包日志：'
try:
if self.logurl_template and len(logfile.parts) >= 2:
# assume the directory name is the time stamp for now.
logurl = string.Template(
self.logurl_template).substitute(
datetime=logfile.parts[-2],
pkgbase=pkgbase)
log_header += ' ' + logurl
except (ValueError,
KeyError): # invalid template or wrong key
pass
msgs.append(log_header)
msgs.append('\n' + build_output)
except FileNotFoundError:
pass
msg = '\n'.join(msgs)
if self.trim_ansi_codes:
msg = ansi_escape_re.sub('', msg)
addresses = [str(x) for x in maintainers]
logger.debug('mail to %s:\nsubject: %s\nbody: %s', addresses,
subject_real, msg[:200])
self.sendmail(addresses, subject_real, msg)
def sendmail(self, who: Union[str, List[str], Maintainer], subject: str,
msg: str) -> None:
if isinstance(who, Maintainer):
who = str(who)
self.ms.sendmail(who, subject, msg)
def send_repo_mail(self, subject: str, msg: str) -> None:
self.ms.sendmail(self.repomail, subject, msg)
def manages(self, dep: Dependency) -> bool:
return dep.pkgdir.name in self.mods
def load_managed_lilac_and_report(self) -> Set[str]:
self.mods, errors = lilacpy.load_managed(self.repodir)
failed = set(errors)
for name, exc_info in errors.items():
tb_lines = traceback.format_exception(*exc_info)
tb = ''.join(tb_lines)
logger.error('error while loading lilac.py for %s',
name,
exc_info=exc_info)
exc = exc_info[1]
if not isinstance(exc, Exception):
raise
self.send_error_report(name,
exc=(exc, tb),
subject='为软件包 %s 载入 lilac.py 时失败')
build_logger_old.error('%s failed', name)
build_logger.exception('lilac.py error',
pkgbase=name,
exc_info=exc_info)
return failed
