def upgrade():
bind = op.get_bind()
session = orm.Session(bind=bind)
db = Redis(
host=getenv("REDIS_SERVICE_HOST", "localhost"),
port=int(getenv("REDIS_SERVICE_PORT", "6379")),
db=0,
decode_responses=True,
)
# tasks
keys = db.keys("celery-task-meta-*")
for key in keys:
data = loads(db.get(key))
task_id = data.get("task_id")
status = data.get("status")
result = data.get("result")
traceback = data.get("traceback")
date_done = data.get("data_done")
if isinstance(date_done, str):
date_done = datetime.fromisoformat(date_done)
logger.info(f"Adding task {task_id} into TaskResultModel")
# our table
TaskResultUpgradeModel.add_task_result(
session=session, task_id=task_id, task_result_dict=result
)
# celery table
add_task_to_celery_table(
session=session,
task_id=task_id,
status=status,
result=result,
traceback=traceback,
date_done=date_done,
)
# whitelist
db = PersistentDict(hash_name="whitelist")
for account, data in db.get_all().items():
if not isinstance(data, dict):
continue
status = data.get("status")
logger.info(f"Adding account {account} into WhitelistModel")
WhitelistUpgradeModel.add_account(
session=session, account_name=account, status=status
)
# installations
for event in RedisInstallation.db().get_all().values():
if not isinstance(event, dict):
continue
event = event["event_data"]
account_login = event.get("account_login")
account_id = event.get("account_id")
account_url = event.get("account_url")
account_type = event.get("account_type")
sender_id = event.get("sender_id")
sender_login = event.get("sender_login")
created_at = event.get("created_at")
if isinstance(created_at, (int, float)):
created_at = datetime.fromtimestamp(created_at, timezone.utc)
elif isinstance(created_at, str):
created_at = created_at.replace("Z", "+00:00")
created_at = datetime.fromisoformat(created_at)
logger.info(f"Adding installation by {account_login} into InstallationModel")
InstallationUpgradeModel.create(
session=session,
account_login=account_login,
account_id=account_id,
account_type=account_type,
account_url=account_url,
sender_login=sender_login,
sender_id=sender_id,
created_at=created_at,
)
# copr-builds
for copr_build in RedisCoprBuild.db().get_all().values():
if not isinstance(copr_build, dict):
continue
project_name = copr_build.get("project")
owner = copr_build.get("owner")
chroots = copr_build.get("chroots")
build_submitted_time = (
datetime.fromisoformat(copr_build.get("build_submitted_time"))
if copr_build.get("build_submitted_time")
else datetime(2020, 1, 1, 0, 0, 0)
)
build_start_time = datetime(2020, 1, 1, 0, 10, 0)
build_finished_time = datetime(2020, 1, 1, 0, 20, 0)
build_id = copr_build.get("build_id")
if not build_id:
continue
status = copr_build.get("status")
web_url = (
f"https://copr.fedorainfracloud.org/coprs/{owner}/{project_name}/"
f"build/{build_id}/"
)
try:
project_name_list = project_name.split("-")
if project_name_list[-1] == "stg":
pr_id = int(project_name_list[-2])
else:
pr_id = int(project_name_list[-1])
job_trigger = JobTriggerUpgradeModel.get_or_create(
type=JobTriggerModelType.pull_request,
trigger_id=pr_id,
session=session,
)
except Exception:
continue
logger.info(f"Adding copr build with build ID {build_id} into CoprBuildModel")
for chroot in chroots:
CoprBuildUpgradeModel.get_or_create(
session=session,
build_id=str(build_id),
project_name=project_name,
owner=owner,
target=chroot,
status=status,
job_trigger=job_trigger,
web_url=web_url,
build_submitted_time=build_submitted_time,
build_start_time=build_start_time,
build_finished_time=build_finished_time,
)
session.commit()
def db(cls) -> PersistentDict:
if not cls.table_name:
raise RuntimeError("table_name is not set")
return PersistentDict(hash_name=cls.table_name)
class Whitelist:
def __init__(self, fas_user: str = None, fas_password: str = None):
self.db = PersistentDict(hash_name="whitelist")
self._fas: AccountSystem = AccountSystem(username=fas_user,
password=fas_password)
def _signed_fpca(self, account_login: str) -> bool:
"""
Check if the user is a packager, by checking if their GitHub
username is in the 'packager' group in FAS. Works only the user's
username is the same in GitHub and FAS.
:param account_login: str, Github username
:return: bool
"""
try:
person = self._fas.person_by_username(account_login)
except AuthError as e:
logger.error(f"FAS authentication failed: {e!r}")
return False
except FedoraServiceError as e:
logger.error(f"FAS query failed: {e!r}")
return False
if not person:
logger.info(f"Not a FAS username {account_login!r}.")
return False
for membership in person.get("memberships", []):
if membership.get("name") == "cla_fpca":
logger.info(f"User {account_login!r} signed FPCA!")
return True
logger.info(f"Cannot verify whether {account_login!r} signed FPCA.")
return False
def get_account(self, account_name: str) -> Optional[dict]:
"""
Get selected account from DB, return None if it's not there
:param account_name: account name for approval
"""
account = self.db.get(account_name)
if not account:
return None
# patch status
db_status = account["status"]
if db_status.startswith("WhitelistStatus"):
account["status"] = db_status.split(".", 1)[1]
self.db[account_name] = account
return account
def add_account(self, github_app: InstallationEvent) -> bool:
"""
Add account to whitelist.
Status is set to 'waiting' or to 'approved_automatically'
if the account is a packager in Fedora.
:param github_app: github app installation info
:return: was the account (auto/already)-whitelisted?
"""
if github_app.account_login in self.db:
# TODO: if the sender added (not created) our App to more repos,
# then we should update the DB here
return True
# Do the DB insertion as a first thing to avoid issue#42
github_app.status = WhitelistStatus.waiting
self.db[github_app.account_login] = github_app.get_dict()
# we want to verify if user who installed the application (sender_login) signed FPCA
# https://fedoraproject.org/wiki/Legal:Fedora_Project_Contributor_Agreement
if self._signed_fpca(github_app.sender_login):
github_app.status = WhitelistStatus.approved_automatically
self.db[github_app.account_login] = github_app.get_dict()
return True
else:
return False
def approve_account(self, account_name: str) -> bool:
"""
Approve user manually
:param account_name: account name for approval
:return:
"""
account = self.get_account(account_name) or {}
account["status"] = WhitelistStatus.approved_manually.value
self.db[account_name] = account
logger.info(f"Account {account_name} approved successfully")
return True
def is_approved(self, account_name: str) -> bool:
"""
Check if user is approved in the whitelist
:param account_name:
:return:
"""
if account_name in self.db:
account = self.get_account(account_name)
db_status = account["status"]
s = WhitelistStatus(db_status)
return (s == WhitelistStatus.approved_automatically
or s == WhitelistStatus.approved_manually)
return False
def remove_account(self, account_name: str) -> bool:
"""
Remove account from whitelist.
:param account_name: github login
:return:
"""
if account_name in self.db:
del self.db[account_name]
# TODO: delete all artifacts from copr
logger.info(f"Account: {account_name} removed from whitelist!")
return True
else:
logger.info(f"Account: {account_name} does not exists!")
return False
def accounts_waiting(self) -> list:
"""
Get accounts waiting for approval
:return: list of accounts waiting for approval
"""
return [
key for (key, item) in self.db.items()
if WhitelistStatus(item["status"]) == WhitelistStatus.waiting
]
def check_and_report(self, event: Optional[Any], project: GitProject,
config: ServiceConfig) -> bool:
"""
Check if account is approved and report status back in case of PR
:param config: service config
:param event: PullRequest and Release TODO: handle more
:param project: GitProject
:return:
"""
# TODO: modify event hierarchy so we can use some abstract classes instead
if isinstance(event, ReleaseEvent):
account_name = event.repo_namespace
if not account_name:
raise KeyError(
f"Failed to get account_name from {type(event)}")
if not self.is_approved(account_name):
logger.info(
f"Refusing release event on not whitelisted repo namespace"
)
return False
return True
if isinstance(
event,
(CoprBuildEvent, TestingFarmResultsEvent, DistGitEvent,
InstallationEvent),
):
return True
if isinstance(event, (PullRequestEvent, PullRequestCommentEvent)):
account_name = event.github_login
if not account_name:
raise KeyError(
f"Failed to get account_name from {type(event)}")
namespace = event.base_repo_namespace
# FIXME:
# Why check account_name when we whitelist namespace only (in whitelist.add_account())?
if not (self.is_approved(account_name)
or self.is_approved(namespace)):
msg = f"Neither account {account_name} nor owner {namespace} are on our whitelist!"
logger.error(msg)
# TODO also check blacklist,
# but for that we need to know who triggered the action
if event.trigger == JobTriggerType.comment:
project.pr_comment(event.pr_id, msg)
else:
job_helper = CoprBuildJobHelper(
config=config,
package_config=event.get_package_config(),
project=project,
event=event,
)
msg = "Account is not whitelisted!" # needs to be shorter
job_helper.report_status_to_all(description=msg,
state="error",
url=FAQ_URL)
return False
# TODO: clear failing check when present
return True
if isinstance(event, IssueCommentEvent):
account_name = event.github_login
if not account_name:
raise KeyError(
f"Failed to get account_name from {type(event)}")
namespace = event.base_repo_namespace
# FIXME:
# Why check account_name when we whitelist namespace only (in whitelist.add_account())?
if not (self.is_approved(account_name)
or self.is_approved(namespace)):
msg = f"Neither account {account_name} nor owner {namespace} are on our whitelist!"
logger.error(msg)
project.issue_comment(event.issue_id, msg)
# TODO also check blacklist,
# but for that we need to know who triggered the action
return False
return True
msg = f"Failed to validate account: Unrecognized event type {type(event)}."
logger.error(msg)
raise PackitException(msg)
def __init__(self, fas_user: str = None, fas_password: str = None):
self.db = PersistentDict(hash_name="whitelist")
self._fas: AccountSystem = AccountSystem(username=fas_user,
password=fas_password)
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from http import HTTPStatus
from logging import getLogger
from flask_restplus import Namespace, Resource
from persistentdict.dict_in_redis import PersistentDict
# hash name is defined in worker (Whitelist class), which I don't want to import from
from packit_service.service.events import Event
db = PersistentDict(hash_name="whitelist")
logger = getLogger("packit_service")
ns = Namespace("whitelist", description="Whitelisted FAS accounts")
@ns.route("")
class WhiteList(Resource):
@ns.response(HTTPStatus.OK, "OK, whitelist follows")
def get(self):
"""List all Whitelisted FAS accounts"""
return [Event.ts2str(event) for event in db.get_all().values()]
@ns.route("/<string:login>")
def __init__(self):
self.db = PersistentDict(hash_name="blacklist")
def task_data(self):
""" get data which celery stores about a task """
return PersistentDict(hash_name=self.identifier)
def get(self, id):
"""A specific copr build details. From copr_build hash, filled by worker."""
# hash name is defined in worker (CoprBuildDB), which I don't want to import from
db = PersistentDict(hash_name="copr_build")
build = db.get(id)
return build if build else ("", HTTPStatus.NO_CONTENT)
def __init__(self):
self.db = PersistentDict(hash_name="whitelist")
class Whitelist:
def __init__(self):
self.db = PersistentDict(hash_name="whitelist")
@staticmethod
def _is_packager(account_login: str) -> bool:
"""
If GitHub username is same as FAS username this method checks if user is packager.
User is considered to be packager when he/she has the badge:
`If you build it... (Koji Success I)`
:param account_login: str, Github username
:return: bool
"""
url = f"https://badges.fedoraproject.org/user/{account_login}/json"
data = requests.get(url)
if not data:
return False
assertions = data.json().get("assertions")
if not assertions:
return False
for item in assertions:
if "Succesfully completed a koji build." in item.get(
"description"):
logger.info(f"User: {account_login} is a packager in Fedora!")
return True
logger.info(
f"Cannot verify whether user: {account_login} is a packager in Fedora."
)
return False
def get_account(self, account_name: str) -> Optional[dict]:
"""
Get selected account from DB, return None if it's not there
:param account_name: account name for approval
"""
account = self.db[account_name]
if not account:
return None
# patch status
db_status = account["status"]
if db_status.startswith("WhitelistStatus"):
account["status"] = db_status.split(".", 1)[1]
self.db[account_name] = account
return account
def add_account(self, github_app: InstallationEvent) -> bool:
"""
Add account to whitelist, if automatic verification of user
(check if user is packager in fedora) fails, account is still inserted in whitelist
with status : `waiting`.
Then a scripts in files/scripts have to be executed for manual approval
:param github_app: github app installation info
:return: was the account auto-whitelisted?
"""
account = self.get_account(github_app.account_login)
if account:
# the account is already in DB
return True
# we want to verify if user who installed the application is packager
if Whitelist._is_packager(github_app.sender_login):
github_app.status = WhitelistStatus.approved_automatically
self.db[github_app.account_login] = github_app.get_dict()
logger.info(f"Account {github_app.account_login} whitelisted!")
return True
else:
logger.error(
"Failed to verify that user is Fedora packager. "
"This could be caused by different github username than FAS username "
"or that user is not a packager.")
github_app.status = WhitelistStatus.waiting
self.db[github_app.account_login] = github_app.get_dict()
logger.info(f"Account {github_app.account_login} inserted "
f"to whitelist with status: waiting for approval")
return False
def approve_account(self, account_name: str) -> bool:
"""
Approve user manually
:param account_name: account name for approval
:return:
"""
account = self.get_account(account_name) or {}
account["status"] = WhitelistStatus.approved_manually.value
self.db[account_name] = account
logger.info(f"Account {account_name} approved successfully")
return True
def is_approved(self, account_name: str) -> bool:
"""
Check if user is approved in the whitelist
:param account_name:
:return:
"""
if account_name in self.db:
account = self.get_account(account_name)
db_status = account["status"]
s = WhitelistStatus(db_status)
return (s == WhitelistStatus.approved_automatically
or s == WhitelistStatus.approved_manually)
return False
def remove_account(self, account_name: str) -> bool:
"""
Remove account from whitelist.
:param account_name: github login
:return:
"""
if account_name in self.db:
del self.db[account_name]
# TODO: delete all artifacts from copr
logger.info(f"User: {account_name} removed from whitelist!")
return True
else:
logger.info(f"User: {account_name} does not exists!")
return False
def accounts_waiting(self) -> list:
"""
Get accounts waiting for approval
:return: list of accounts waiting for approval
"""
return [
key for (key, item) in self.db.items()
if WhitelistStatus(item["status"]) == WhitelistStatus.waiting
]
def check_and_report(self, event: Optional[Any],
project: GitProject) -> bool:
"""
Check if account is approved and report status back in case of PR
:param event: PullRequest and Release TODO: handle more
:param project: GitProject
:return:
"""
account_name = None
if isinstance(event, PullRequestEvent):
account_name = event.base_repo_namespace
if isinstance(event, ReleaseEvent):
account_name = event.repo_namespace
if account_name:
if not self.is_approved(account_name):
logger.error(
f"User {account_name} is not approved on whitelist!")
# TODO also check blacklist,
# but for that we need to know who triggered the action
if event.trigger == JobTriggerType.pull_request:
r = BuildStatusReporter(project, event.commit_sha, None)
msg = "Account is not whitelisted!"
r.report(
"failure",
msg,
url=FAQ_URL,
check_name=PRCheckName.get_build_check(),
)
return False
return True
def db(self):
return PersistentDict(hash_name=str(uuid4()))