async def _check_reposet(
request: AthenianWebRequest,
sdb_conn: Optional[databases.core.Connection],
account: int,
body: List[str],
) -> List[str]:
service = None
repos = set()
for repo in body:
for key, prefix in PREFIXES.items():
if repo.startswith(prefix):
if service is None:
service = key
elif service != key:
raise ResponseError(
BadRequestError(detail="mixed services: %s, %s" %
(service, key), ))
repos.add(repo[len(prefix):])
if service is None:
raise ResponseError(
BadRequestError(
detail=
"repository prefixes do not match to any supported service", ))
checker = await access_classes[service](account, sdb_conn, request.mdb,
request.cache).load()
denied = await checker.check(repos)
if denied:
raise ResponseError(
ForbiddenError(
detail="the following repositories are access denied for %s: %s"
% (service, denied), ))
return sorted(set(body))
async def fetch_reposet(
id: int,
columns: Union[Sequence[Type[RepositorySet]],
Sequence[InstrumentedAttribute]],
uid: str,
sdb: Union[databases.Database, databases.core.Connection],
cache: Optional[aiomcache.Client],
) -> Tuple[RepositorySet, bool]:
"""
Retrieve a repository set by ID and check the access for the given user.
:return: Loaded RepositorySet and `is_admin` flag that indicates whether the user has \
RW access to that set.
"""
if not columns or columns[0] is not RepositorySet:
for col in columns:
if col is RepositorySet.owner:
break
else:
columns = list(columns)
columns.append(RepositorySet.owner)
rs = await sdb.fetch_one(select(columns).where(RepositorySet.id == id))
if rs is None or len(rs) == 0:
raise ResponseError(
NotFoundError(detail="Repository set %d does not exist" % id))
account = rs[RepositorySet.owner.key]
adm = await get_user_account_status(uid, account, sdb, cache)
return RepositorySet(**rs), adm
async def create_reposet(request: AthenianWebRequest,
body: dict) -> web.Response:
"""Create a repository set.
:param body: List of repositories to group.
"""
body = RepositorySetCreateRequest.from_dict(body)
user = request.uid
account = body.account
async with request.sdb.connection() as sdb_conn:
try:
adm = await get_user_account_status(user, account, sdb_conn,
request.cache)
except ResponseError as e:
return e.response
if not adm:
return ResponseError(
ForbiddenError(
detail="User %s is not an admin of the account %d" %
(user, account))).response
try:
items = await _check_reposet(request, sdb_conn, body.account,
body.items)
except ResponseError as e:
return e.response
rs = RepositorySet(owner=account, items=items).create_defaults()
rid = await sdb_conn.execute(
insert(RepositorySet).values(rs.explode()))
return response(CreatedIdentifier(rid))
async def create_new_reposet(_mdb_conn: databases.core.Connection):
# a new account, discover their repos from the installation and create the first reposet
iid = await get_installation_id(account, sdb_conn, cache)
if iid is None:
iid = await _mdb_conn.fetch_val(
select([InstallationOwner.install_id]).where(
InstallationOwner.user_id == int(native_uid)).order_by(
InstallationOwner.created_at.desc()))
if iid is None:
raise ResponseError(
NoSourceDataError(
detail=
"The metadata installation has not registered yet."))
await sdb_conn.execute(
update(Account).where(Account.id == account).values(
{Account.installation_id.key: iid}))
repos = await _mdb_conn.fetch_all(
select([InstallationRepo.repo_full_name
]).where(InstallationRepo.install_id == iid))
repos = [("github.com/" + r[InstallationRepo.repo_full_name.key])
for r in repos]
rs = RepositorySet(owner=account, items=repos).create_defaults()
rs.id = await sdb_conn.execute(
insert(RepositorySet).values(rs.explode()))
logging.getLogger(__package__).info(
"Created the first reposet %d for account %d with %d repos on behalf of %s",
rs.id,
account,
len(repos),
native_uid,
)
return [vars(rs)]
async def update_reposet(request: AthenianWebRequest, id: int,
body: List[str]) -> web.Response:
"""Update a repository set.
:param id: Numeric identifier of the repository set to update.
:type id: int
:param body: New list of repositories in the group.
"""
async with request.sdb.connection() as sdb_conn:
try:
rs, is_admin = await fetch_reposet(id, [RepositorySet],
request.uid, sdb_conn,
request.cache)
except ResponseError as e:
return e.response
if not is_admin:
return ResponseError(
ForbiddenError(detail="User %s may not modify reposet %d" %
(request.uid, id))).response
try:
body = await _check_reposet(request, sdb_conn, id, body)
except ResponseError as e:
return e.response
rs.items = body
rs.refresh()
await sdb_conn.execute(
update(RepositorySet).where(RepositorySet.id == id).values(
rs.explode()))
return web.json_response(body, status=200)
async def _get_user_info_cached(self, token: str) -> User:
resp = await self._session.get(
"https://%s/userinfo" % self._domain,
headers={"Authorization": "Bearer " + token})
try:
user = await resp.json()
except aiohttp.ContentTypeError:
raise ResponseError(
GenericError("/errors/Auth0",
title=resp.reason,
status=resp.status,
detail=await resp.text()))
if resp.status != 200:
raise ResponseError(
GenericError("/errors/Auth0",
title=resp.reason,
status=resp.status,
detail=user.get("description", str(user))))
return User.from_auth0(**user)
async def _resolve_repos(
filt: Union[FilterContribsOrReposRequest, FilterPullRequestsRequest],
uid: str,
native_uid: str,
sdb_conn: Union[databases.core.Connection, databases.Database],
mdb_conn: Union[databases.core.Connection, databases.Database],
cache: Optional[aiomcache.Client],
) -> List[str]:
status = await sdb_conn.fetch_one(
select([UserAccount.is_admin]).where(
and_(UserAccount.user_id == uid,
UserAccount.account_id == filt.account)))
if status is None:
raise ResponseError(
ForbiddenError(detail="User %s is forbidden to access account %d" %
(uid, filt.account)))
check_access = True
if not filt.in_:
rss = await load_account_reposets(filt.account, native_uid,
[RepositorySet.id], sdb_conn,
mdb_conn, cache)
filt.in_ = ["{%d}" % rss[0][RepositorySet.id.key]]
check_access = False
repos = set(
chain.from_iterable(await asyncio.gather(*[
resolve_reposet(r, ".in[%d]" %
i, uid, filt.account, sdb_conn, cache)
for i, r in enumerate(filt.in_)
])))
prefix = "github.com/"
repos = [r[r.startswith(prefix) and len(prefix):] for r in repos]
if check_access:
checker = await access_classes["github"](filt.account, sdb_conn,
mdb_conn, cache).load()
denied = await checker.check(set(repos))
if denied:
raise ResponseError(
ForbiddenError(
detail=
"the following repositories are access denied for %s: %s" %
("github.com/", denied), ))
return repos
async def get_installation_id(
account: int,
sdb_conn: Union[databases.Database, databases.core.Connection],
cache: Optional[aiomcache.Client],
) -> int:
"""Fetch the Athenian metadata installation ID for the given account ID."""
iid = await sdb_conn.fetch_val(
select([Account.installation_id]).where(Account.id == account))
if iid is None:
raise ResponseError(
NoSourceDataError(
detail="The account installation has not finished yet."))
return iid
async def resolve_reposet(
repo: str,
pointer: str,
uid: str,
account: int,
db: Union[databases.core.Connection, databases.Database],
cache: Optional[aiomcache.Client],
) -> List[str]:
"""
Dereference the repository sets.
If `repo` is a regular repository, return `[repo]`. Otherwise, return the list of \
repositories by the parsed ID from the database.
"""
if not repo.startswith("{"):
return [repo]
if not repo.endswith("}"):
raise ResponseError(
InvalidRequestError(
detail="repository set format is invalid: %s" % repo,
pointer=pointer,
))
try:
set_id = int(repo[1:-1])
except ValueError:
raise ResponseError(
InvalidRequestError(
detail="repository set identifier is invalid: %s" % repo,
pointer=pointer,
))
rs, _ = await fetch_reposet(set_id, [RepositorySet.items], uid, db, cache)
if rs.owner != account:
raise ResponseError(
ForbiddenError(
detail=
"User %s is not allowed to reference reposet %d in this query"
% (uid, set_id)))
return rs.items
async def become_user(request: AthenianWebRequest,
id: str = "") -> web.Response:
"""God mode ability to turn into any user. The current user must be marked internally as \
a super admin."""
user_id = getattr(request, "god_id", None)
if user_id is None:
return ResponseError(
ForbiddenError(detail="User %s is not allowed to mutate" %
user_id)).response
async with request.sdb.connection() as conn:
if id and (await conn.fetch_one(
select([UserAccount]).where(UserAccount.user_id == id)
)) is None:
return ResponseError(
NotFoundError(detail="User %s does not exist" % id)).response
god = await conn.fetch_one(select([God]).where(God.user_id == user_id))
god = God(**god).refresh()
god.mapped_id = id or None
await conn.execute(
update(God).where(God.user_id == user_id).values(god.explode()))
user = await (await
request.auth.get_user(id
or user_id)).load_accounts(request.sdb)
return response(user)
async def get_user_account_status(
user: str,
account: int,
sdb_conn: Union[databases.Database, databases.core.Connection],
cache: Optional[aiomcache.Client],
) -> bool:
"""Return the value indicating whether the given user is an admin of the given account."""
status = await sdb_conn.fetch_val(
select([UserAccount.is_admin]).where(
and_(UserAccount.user_id == user,
UserAccount.account_id == account)))
if status is None:
raise ResponseError(
NotFoundError(
detail="Account %d does not exist or user %s is not a member."
% (account, user)))
return status
async def delete_reposet(request: AthenianWebRequest, id: int) -> web.Response:
"""Delete a repository set.
:param id: Numeric identifier of the repository set to delete.
:type id: int
"""
try:
_, is_admin = await fetch_reposet(id, [], request.uid, request.sdb,
request.cache)
except ResponseError as e:
return e.response
if not is_admin:
return ResponseError(
ForbiddenError(detail="User %s may not modify reposet %d" %
(request.uid, id))).response
await request.sdb.execute(
delete(RepositorySet).where(RepositorySet.id == id))
return web.Response(status=200)
async def get_account(request: AthenianWebRequest, id: int) -> web.Response:
"""Return details about the current account."""
user_id = request.uid
users = await request.sdb.fetch_all(
select([UserAccount]).where(UserAccount.account_id == id))
for user in users:
if user[UserAccount.user_id.key] == user_id:
break
else:
return ResponseError(
ForbiddenError(
detail="User %s is not allowed to access account %d" %
(user_id, id))).response
admins = []
regulars = []
for user in users:
role = admins if user[UserAccount.is_admin.key] else regulars
role.append(user[UserAccount.user_id.key])
users = await request.auth.get_users(regulars + admins)
account = Account(regulars=[users[k] for k in regulars if k in users],
admins=[users[k] for k in admins if k in users])
return response(account)
async def with_db(self, request, handler):
"""Add "mdb" and "sdb" attributes to every incoming request."""
if self.mdb is None:
await self._mdb_future
assert self.mdb is not None
del self._mdb_future
if self.sdb is None:
await self._sdb_future
assert self.sdb is not None
del self._sdb_future
request.mdb = self.mdb
request.sdb = self.sdb
request.cache = self._cache
try:
return await handler(request)
except ConnectionError as e:
return ResponseError(
GenericError(
type="/errors/InternalConnectivityError",
title=HTTPStatus.SERVICE_UNAVAILABLE.phrase,
status=HTTPStatus.SERVICE_UNAVAILABLE,
detail="%s: %s" % (type(e).__name__, e),
)).response
async def calc_metrics_pr_linear(request: AthenianWebRequest,
body: dict) -> web.Response:
"""Calculate linear metrics over PRs.
:param request: HTTP request.
:param body: Desired metric definitions.
:type body: dict | bytes
"""
try:
body = PullRequestMetricsRequest.from_dict(body)
except ValueError as e:
return ResponseError(InvalidRequestError("?", detail=str(e))).response
"""
@se7entyse7en:
It seems weird to me that the generated class constructor accepts None as param and it
doesn't on setters. Probably it would have much more sense to generate a class that doesn't
accept the params at all or that it does not default to None. :man_shrugging:
@vmarkovtsev:
This is exactly what I did the other day. That zalando/connexion thingie which glues OpenAPI
and asyncio together constructs all the models by calling their __init__ without any args and
then setting individual attributes. So we crash somewhere in from_dict() or to_dict() if we
make something required.
"""
met = CalculatedMetrics()
met.date_from = body.date_from
met.date_to = body.date_to
met.granularity = body.granularity
met.metrics = body.metrics
met.calculated = []
try:
filters = await _compile_filters(body.for_, request, body.account)
except ResponseError as e:
return e.response
if body.date_to < body.date_from:
return ResponseError(
InvalidRequestError(
detail="date_from may not be greater than date_to",
pointer=".date_from",
)).response
try:
time_intervals = Granularity.split(body.granularity, body.date_from,
body.date_to)
except ValueError:
return ResponseError(
InvalidRequestError(
detail="granularity value is invalid",
pointer=".granularity",
)).response
for service, (repos, devs, for_set) in filters:
calcs = defaultdict(list)
# for each filter, we find the functions to measure the metrics
sentries = METRIC_ENTRIES[service]
for m in body.metrics:
calcs[sentries[m]].append(m)
results = {}
# for each metric, we find the function to calculate and call it
for func, metrics in calcs.items():
fres = await func(metrics, time_intervals, repos, devs,
request.mdb, request.cache)
assert len(fres) == len(time_intervals) - 1
for i, m in enumerate(metrics):
results[m] = [r[i] for r in fres]
cm = CalculatedMetric(
for_=for_set,
values=[
CalculatedMetricValues(
date=d,
values=[results[m][i].value for m in met.metrics],
confidence_mins=[
results[m][i].confidence_min for m in met.metrics
],
confidence_maxs=[
results[m][i].confidence_max for m in met.metrics
],
confidence_scores=[
results[m][i].confidence_score() for m in met.metrics
],
) for i, d in enumerate(time_intervals[1:])
])
for v in cm.values:
if sum(1 for c in v.confidence_scores if c is not None) == 0:
v.confidence_mins = None
v.confidence_maxs = None
v.confidence_scores = None
met.calculated.append(cm)
return response(met)
async def _compile_filters(
for_sets: List[ForSet],
request: AthenianWebRequest,
account: int,
) -> List[Filter]:
filters = []
sdb, user = request.sdb, request.uid
checkers = {}
async with sdb.connection() as sdb_conn:
for i, for_set in enumerate(for_sets):
repos = set()
devs = []
service = None
for repo in chain.from_iterable(await asyncio.gather(*[
resolve_reposet(r, ".for[%d].repositories[%d]" %
(i, j), user, account, sdb, request.cache)
for j, r in enumerate(for_set.repositories)
])):
for key, prefix in PREFIXES.items():
if repo.startswith(prefix):
if service is None:
service = key
elif service != key:
raise ResponseError(
InvalidRequestError(
detail=
'mixed providers are not allowed in the same "for" element',
pointer=".for[%d].repositories" % i,
))
repos.add(repo[len(prefix):])
if service is None:
raise ResponseError(
InvalidRequestError(
detail=
'the provider of a "for" element is unsupported or the set is empty',
pointer=".for[%d].repositories" % i,
))
checker = checkers.get(service)
if checker is None:
checker = await access_classes[service](account, sdb_conn,
request.mdb,
request.cache).load()
checkers[service] = checker
denied = await checker.check(repos)
if denied:
raise ResponseError(
InvalidRequestError(
detail=
"the following repositories are access denied for %s: %s"
% (service, denied),
pointer=".for[%d].repositories" % i,
status=HTTPStatus.FORBIDDEN,
))
for dev in (for_set.developers or []):
for key, prefix in PREFIXES.items():
if dev.startswith(prefix):
if service != key:
raise ResponseError(
InvalidRequestError(
detail=
'mixed providers are not allowed in the same "for" element',
pointer=".for[%d].developers" % i,
))
devs.append(dev[len(prefix):])
filters.append((service, (repos, devs, for_set)))
return filters