def test_process_success_waiting_for_package_different_revision(self):
build_output = _setup_build_output("create.json")
with database.session_scope() as session:
ecosystem = util.create_ecosystem(dict())
waiting_build_id = _create_waiting_build(
session,
ecosystem,
missing_packages=[
util.create_package({
"ecosystem":
ecosystem,
"recipe_revision.revision":
"f5c1ba6f1af634f500f7e0255619fecf4777965f"
})
])
build_id = _create_build(session, ecosystem)
result = manager.process_success(build_id, build_output)
self.assertTrue(result["new_builds"])
with database.session_scope() as session:
waiting_build = session.query(
database.Build).filter_by(id=waiting_build_id).first()
self.assertEqual(database.BuildStatus.new, waiting_build.status)
def test_process_success_waiting_for_package_different_package_id(self):
build_output = _setup_build_output("create.json")
with database.session_scope() as session:
ecosystem = util.create_ecosystem(dict())
waiting_build_id = _create_waiting_build(
session,
ecosystem,
missing_packages=[
util.create_package({
"ecosystem":
ecosystem,
"package.package_id":
"d057732059ea44a47760900cb5e4855d2bea8714"
})
])
build_id = _create_build(session, ecosystem)
result = manager.process_success(build_id, build_output)
self.assertFalse("new_builds" in result.keys())
with database.session_scope() as session:
waiting_build = session.query(
database.Build).filter_by(id=waiting_build_id).first()
self.assertEqual(database.BuildStatus.error, waiting_build.status)
def test_process_failure_missing_package(self):
build_output = _setup_build_output("create_missing_package.json")
with database.session_scope() as session:
build = util.create_build(dict())
session.add(build)
session.commit()
build_id = build.id
manager.process_failure(build_id, build_output)
with database.session_scope() as session:
build = session.query(
database.Build).filter_by(id=build_id).first()
self.assertEqual(1, len(build.missing_packages))
package = build.missing_packages[0]
self.assertEqual("d057732059ea44a47760900cb5e4855d2bea8714",
package.package_id)
recipe_revision = package.recipe_revision
self.assertIsNotNone(recipe_revision)
self.assertEqual("f5c1ba6f1af634f500f7e0255619fecf4777965f",
recipe_revision.revision)
recipe = recipe_revision.recipe
self.assertIsNotNone(recipe)
self.assertEqual("base", recipe.name)
self.assertEqual("1.2.3", recipe.version)
self.assertEqual("mycompany", recipe.user)
self.assertEqual("stable", recipe.channel)
async def __process_commits(self):
logger.info("Start processing commits")
new_commits = False
with database.session_scope() as session:
commits = session.query(
database.Commit).filter_by(status=database.CommitStatus.new)
profiles = session.query(database.Profile).all()
for commit in commits:
logger.info("Process commit '%s' of repo '%s'", commit.sha[:7],
commit.repo.url)
exclude_labels = {label.value for label in commit.repo.exclude}
for profile in profiles:
labels = {label.value for label in profile.labels}
if not labels.isdisjoint(exclude_labels):
logger.info("Exclude build for '%s' with profile '%s'",
commit.sha[:7], profile.name)
continue
new_commits = True
logger.info("Schedule build for '%s' with profile '%s'",
commit.sha[:7], profile.name)
build = database.Build()
build.profile = profile
build.commit = commit
build.status = database.BuildStatus.new
build.log = database.Log()
build.log.logs = ''
session.add(build)
logger.info("Set commit '%s' to 'building'", commit.sha[:7])
commit.status = database.CommitStatus.building
if new_commits:
logger.info("Finish processing commits with *new* builds")
else:
logger.info("Finish processing commits with *no* builds")
with database.session_scope() as session:
num_new_builds = session.query(database.Build).filter_by(
status=database.BuildStatus.new).count()
logger.info("Currently %d new builds exist", num_new_builds)
if num_new_builds == 0:
return new_commits
logger.info('Trigger linux agent: process builds')
try:
self.__linux_agent.process_builds()
except (ApiException, MaxRetryError):
logger.error("Failed to trigger Linux agent")
logger.info('Trigger windows agent: process builds')
try:
self.__windows_agent.process_builds()
except (ApiException, MaxRetryError):
logger.error("Failed to trigger Windows agent")
return new_commits
def test_start_invalid_repo(self):
with database.session_scope() as session:
session.add(util.create_repo({"repo.invalid": True}))
session.add(util.create_channel(dict()))
self.crawler.start()
time.sleep(3)
with database.session_scope() as session:
commits = session.query(database.Commit).all()
self.assertEqual(len(commits), 0)
def test_stop_build(self):
with database.session_scope() as session:
build = util.create_build({"repo.deadlock": True})
session.add(build)
self.agent.start()
self.__wait_for_build_status(database.BuildStatus.active, 15)
with database.session_scope() as session:
build = session.query(database.Build).first()
build.status = database.BuildStatus.stopping
self.assertEqual(
self.__wait_for_build_status(database.BuildStatus.stopped, 15),
database.BuildStatus.stopped)
def test_create_build_with_missing_recipe(self):
with database.session_scope() as session:
build = util.create_build(dict())
recipe = util.create_recipe(dict())
build.missing_recipes = [recipe]
session.add(build)
with database.session_scope() as session:
num_recipes = session.query(database.Recipe).count()
self.assertEqual(1, num_recipes)
recipe = session.query(database.Recipe).first()
self.assertEqual(1, len(recipe.waiting_builds))
def test_start_repo_and_regex_channel(self):
with database.session_scope() as session:
session.add(util.create_repo(dict()))
session.add(util.create_channel({"channel.branch": "mas.*"}))
self.crawler.start()
time.sleep(5)
called = self.crawler.query(
lambda: self.scheduler.process_commits.called)
self.assertTrue(called)
with database.session_scope() as session:
commit = session.query(database.Commit).first()
self.assertEqual(database.CommitStatus.new, commit.status)
def test_create_build_with_missing_package(self):
with database.session_scope() as session:
build = util.create_build(dict())
package = util.create_package(dict())
build.missing_packages = [package]
session.add(build)
with database.session_scope() as session:
num_packages = session.query(database.Package).count()
self.assertEqual(1, num_packages)
package = session.query(database.Package).first()
self.assertEqual(1, len(package.waiting_builds))
def test_remove_but_last_user_last_user(self):
with database.session_scope() as session:
user = database.User()
user.user_name = "user"
session.add(user)
self.assertRaises(database.OperationFailed,
lambda: database.remove_but_last_user("1"))
with database.session_scope() as session:
user = session.query(database.User).filter_by(id="1").first()
self.assertIsNotNone(user)
def test_http_repo(self):
with database.session_scope() as session:
session.add(util.create_repo({"repo.https": True}))
session.add(util.create_channel(dict()))
self.crawler.start()
time.sleep(5)
called = self.crawler.query(
lambda: self.scheduler.process_commits.called)
self.assertTrue(called)
with database.session_scope() as session:
commit = session.query(database.Commit).first()
self.assertEqual(database.CommitStatus.new, commit.status)
def test_complete_build_with_dependency(self):
with database.session_scope() as session:
session.add(
util.create_build({
"repo.dependent": True,
"ecosystem.empty_remote": True
}))
self.agent.start()
self.assertEquals(
self.__wait_for_build_status(database.BuildStatus.error, 15),
database.BuildStatus.error)
with database.session_scope() as session:
build = session.query(database.Build).first()
self.assertEquals(1, len(build.missing_recipes))
def test_remove_but_last_user(self):
with database.session_scope() as session:
user1 = database.User()
user2 = database.User()
user1.user_name = "user1"
user2.user_name = "user2"
session.add(user1)
session.add(user2)
# should not raise an exception
database.remove_but_last_user("1")
with database.session_scope() as session:
num_users = session.query(database.User).count()
self.assertEqual(num_users, 1)
def get_package(package_id):
with database.session_scope() as session:
record = session.query(
database.Package).filter_by(id=package_id).first()
if not record:
abort(404)
return models.PackageData(data=__create_package(record))
def update_build(build_id, body=None):
if connexion.request.is_json:
body = models.BuildData.from_dict(
connexion.request.get_json()) # noqa: E501
with database.session_scope() as session:
record = session.query(database.Build).filter_by(id=build_id).first()
if not record:
abort(404)
try:
record.status = build_status_table[body.data.attributes.status]
if record.status == database.BuildStatus.new:
record.log.logs = ''
session.commit()
logger.info('Trigger linux agent: process builds')
try:
linux_agent.process_builds()
except MaxRetryError:
logger.error("Failed to trigger Linux agent")
logger.info('Trigger windows agent: process builds')
try:
windows_agent.process_builds()
except MaxRetryError:
logger.error("Failed to trigger Windows agent")
except KeyError:
abort(400)
return models.BuildData(data=__create_build(record))
def delete_repo(repo_id):
with database.session_scope() as session:
record = session.query(database.Repo).filter_by(id=repo_id).first()
if not record:
abort(404)
session.delete(record)
return None
def add_ecosystem(body=None):
if connexion.request.is_json:
body = models.EcosystemData.from_dict(
connexion.request.get_json()) # noqa: E501
record = database.Ecosystem()
record.name = body.data.attributes.name
record.user = body.data.attributes.user
record.conan_config_url = body.data.attributes.conan_config_url
record.conan_config_branch = body.data.attributes.conan_config_branch
record.conan_config_path = body.data.attributes.conan_config_path
record.conan_remote = body.data.attributes.conan_remote
record.conan_user = body.data.attributes.conan_user
record.conan_password = body.data.attributes.conan_password
record.known_hosts = body.data.attributes.known_hosts
private, public = generate_rsa_key()
record.ssh_key = encode(private)
record.public_ssh_key = encode(public)
credentials = []
for c in body.data.attributes.credentials:
git_credential = database.GitCredential()
git_credential.url = c.url
git_credential.username = c.username
git_credential.password = c.password
credentials.append(git_credential)
record.credentials = credentials
with database.session_scope() as session:
session.add(record)
session.commit()
return models.EcosystemData(data=__create_ecosystem(record)), 201
def test_complete_build_https(self):
with database.session_scope() as session:
session.add(util.create_build({"repo.https": True}))
self.agent.start()
self.assertEquals(
self.__wait_for_build_status(database.BuildStatus.success, 15),
database.BuildStatus.success)
def test_start_build(self):
with database.session_scope() as session:
session.add(util.create_build(dict()))
self.agent.start()
self.assertEquals(
self.__wait_for_build_status(database.BuildStatus.active, 15),
database.BuildStatus.active)
def get_ecosystem(ecosystem_id):
with database.session_scope() as session:
record = session.query(
database.Ecosystem).filter_by(id=ecosystem_id).first()
if not record:
abort(404)
return models.EcosystemData(data=__create_ecosystem(record))
def get_channel(channel_id):
with database.session_scope() as session:
record = session.query(
database.Channel).filter_by(id=channel_id).first()
if not record:
abort(404)
return models.ChannelData(data=__createChannel(record))
def update_ecosystem(ecosystem_id, body=None):
if connexion.request.is_json:
body = models.EcosystemData.from_dict(
connexion.request.get_json()) # noqa: E501
with database.session_scope() as session:
record = session.query(
database.Ecosystem).filter_by(id=ecosystem_id).first()
if not record:
abort(404)
record.name = body.data.attributes.name
record.user = body.data.attributes.user
record.conan_config_url = body.data.attributes.conan_config_url
record.conan_config_branch = body.data.attributes.conan_config_branch
record.conan_config_path = body.data.attributes.conan_config_path
record.conan_remote = body.data.attributes.conan_remote
record.conan_user = body.data.attributes.conan_user
record.conan_password = body.data.attributes.conan_password
record.known_hosts = body.data.attributes.known_hosts
credentials = []
for c in body.data.attributes.credentials:
git_credential = database.GitCredential()
git_credential.url = c.url
git_credential.username = c.username
git_credential.password = c.password
credentials.append(git_credential)
record.credentials = credentials
if body.data.attributes.public_ssh_key == '':
private, public = generate_rsa_key()
record.ssh_key = encode(private)
record.public_ssh_key = encode(public)
return models.EcosystemData(data=__create_ecosystem(record))
def add_user(body=None):
if connexion.request.is_json:
body = models.UserData.from_dict(
connexion.request.get_json()) # noqa: E501
if not body.data.attributes.user_name:
abort(400)
record = database.User()
record.user_name = body.data.attributes.user_name
record.last_name = body.data.attributes.last_name
record.first_name = body.data.attributes.first_name
record.email = body.data.attributes.email
if body.data.attributes.password:
record.password = hash_password(body.data.attributes.password)
record.permissions.clear()
if body.data.attributes.permissions:
for p in body.data.attributes.permissions:
permission = database.Permission(
permission_label_table[p.permission])
record.permissions.append(permission)
try:
with database.session_scope() as session:
session.add(record)
session.commit()
return models.UserData(data=__create_user(record)), 201
except sqlalchemy.exc.IntegrityError as e:
if e.orig.args[0] == database.ErrorCodes.DUPLICATE_ENTRY:
abort(409)
abort(400)
def add_profile(body=None):
if connexion.request.is_json:
body = models.ProfileData.from_dict(
connexion.request.get_json()) # noqa: E501
with database.session_scope() as session:
ecosystem = session.query(database.Ecosystem)\
.filter_by(id=body.data.relationships.ecosystem.data.id)\
.first()
if not ecosystem:
abort(400)
record = database.Profile()
record.name = body.data.attributes.name
record.platform = platform_table[body.data.attributes.platform]
record.ecosystem = ecosystem
record.container = body.data.attributes.container
record.conan_profile = body.data.attributes.conan_profile
record.docker_user = body.data.attributes.docker_user
record.docker_password = body.data.attributes.docker_password
record.labels = [
database.Label(l.label) for l in body.data.attributes.labels
]
session.add(record)
session.commit()
return models.ProfileData(data=__create_profile(record)), 201
def get_commits(repo_id):
with database.session_scope() as session:
return models.CommitList(data=[
__create_commit(record)
for record in session.query(database.Commit).filter_by(
repo_id=repo_id).all()
])
def test_start_repo_but_no_channel(self):
with database.session_scope() as session:
session.add(util.create_repo(dict()))
self.crawler.start()
called = self.crawler.query(
lambda: self.scheduler.process_commits.called)
self.assertFalse(called)
def add_repo(body=None):
if connexion.request.is_json:
body = models.RepoData.from_dict(
connexion.request.get_json()) # noqa: E501
with database.session_scope() as session:
ecosystem = session.query(database.Ecosystem)\
.filter_by(id=body.data.relationships.ecosystem.data.id)\
.first()
if not ecosystem:
abort(400)
record = database.Repo()
record.name = body.data.attributes.name
record.path = body.data.attributes.path
record.url = body.data.attributes.url
record.exclude = [
database.Label(l.label) for l in body.data.attributes.exclude
if l.label
]
record.options = [
database.Option(o.key, o.value)
for o in body.data.attributes.options if o.key
]
record.ecosystem = ecosystem
session.add(record)
session.commit()
return models.RepoData(data=__create_repo(record)), 201
def get_recipe_revision(recipe_revision_id):
with database.session_scope() as session:
record = session.query(
database.RecipeRevision).filter_by(id=recipe_revision_id).first()
if not record:
abort(404)
return models.RecipeRevisionData(data=__create_recipe_revision(record))
def get_repos(ecosystem_id):
with database.session_scope() as session:
return models.RepoList(data=[
__create_repo(record)
for record in session.query(database.Repo).filter_by(
ecosystem_id=ecosystem_id).all()
])
def login(body=None): # noqa: E501
"""log in
# noqa: E501
:param body: login credentials
:type body: dict | bytes
:rtype: None
"""
if connexion.request.is_json:
body = models.Credentials.from_dict(
connexion.request.get_json()) # noqa: E501
with database.session_scope() as session:
record = session.query(
database.User).filter_by(user_name=body.user_name).first()
if not record or not record.password:
abort(401, 'Wrong credentials')
if not test_password(body.password, record.password):
abort(401, 'Wrong credentials')
user = auth.User(str(record.id))
login_user(user)
return models.UserToken(user_id=auth.get_user()), 200