def test_aggregate_repo_files_hashed_dir(self):
utils.aggregate_repo_files('test1',
self.datadir,
self.session,
'delorean',
hashed_dir=True)
expected_file = os.path.join(self.datadir, 'repos', 'test1',
'delorean.repo')
assert os.path.exists(expected_file)
assert os.path.islink(expected_file)
with open(expected_file, 'r') as fp:
contents = fp.read()
assert contents == 'TESTING ONE TWO THREE\n'
def test_aggregate_repo_files(self):
result = utils.aggregate_repo_files('test1', self.datadir,
self.session, 'delorean')
expected_file = os.path.join(self.datadir, 'repos', 'test1',
'delorean.repo')
assert os.path.exists(expected_file)
with open(expected_file, 'r') as fp:
contents = fp.read()
assert contents == 'TESTING ONE TWO THREE\n'
assert result == hashlib.md5(b'TESTING ONE TWO THREE\n').hexdigest()
def promote():
# commit_hash: commit hash
# distro_hash: distro hash
# extended_hash (optional): extended hash
# promote_name: symlink name
try:
commit_hash = request.json['commit_hash']
distro_hash = request.json['distro_hash']
promote_name = request.json['promote_name']
except KeyError:
raise InvalidUsage('Missing parameters', status_code=400)
extended_hash = request.json.get('extended_hash', None)
# Check for invalid promote names
if (promote_name == 'consistent' or promote_name == 'current'):
raise InvalidUsage('Invalid promote_name %s' % promote_name,
status_code=403)
config_options = _get_config_options(app.config['CONFIG_FILE'])
session = _get_db()
commit = _get_commit(session, commit_hash, distro_hash, extended_hash)
if commit is None:
raise InvalidUsage('commit_hash+distro_hash+extended_hash combination'
' not found', status_code=404)
# If the commit has been purged, do not move on
if commit.flags & FLAG_PURGED:
raise InvalidUsage('commit_hash+distro_hash+extended_hash has been '
'purged, cannot promote it', status_code=410)
if config_options.use_components:
base_directory = os.path.join(app.config['REPO_PATH'], "component/%s" %
commit.component)
else:
base_directory = app.config['REPO_PATH']
target_link = os.path.join(base_directory, promote_name)
# Check for invalid target links, like ../promotename
target_dir = os.path.dirname(os.path.abspath(target_link))
if not os.path.samefile(target_dir, base_directory):
raise InvalidUsage('Invalid promote_name %s' % promote_name,
status_code=403)
# We should create a relative symlink
yumrepodir = commit.getshardedcommitdir()
if config_options.use_components:
# In this case, the relative path should not include
# the component part
yumrepodir = yumrepodir.replace("component/%s/" % commit.component, '')
# Remove symlink if it exists, so we can create it again
if os.path.lexists(os.path.abspath(target_link)):
os.remove(target_link)
try:
os.symlink(yumrepodir, target_link)
except Exception as e:
raise InvalidUsage("Symlink creation failed with error: %s" %
e, status_code=500)
# Once the updated symlink is created, if we are using components
# we need to update the top-level repo file
repo_checksum = None
if config_options.use_components:
datadir = os.path.realpath(config_options.datadir)
repo_checksum = aggregate_repo_files(promote_name, datadir, session,
config_options.reponame,
hashed_dir=True)
timestamp = time.mktime(datetime.now().timetuple())
promotion = Promotion(commit_id=commit.id, promotion_name=promote_name,
timestamp=timestamp, user=auth.username(),
component=commit.component,
aggregate_hash=repo_checksum)
session.add(promotion)
session.commit()
repo_hash = _repo_hash(commit)
repo_url = "%s/%s" % (config_options.baseurl, commit.getshardedcommitdir())
result = {'commit_hash': commit_hash,
'distro_hash': distro_hash,
'extended_hash': commit.extended_hash,
'repo_hash': repo_hash,
'repo_url': repo_url,
'promote_name': promote_name,
'component': commit.component,
'timestamp': timestamp,
'user': auth.username(),
'aggregate_hash': repo_checksum}
return jsonify(result), 201
def promote_batch():
# hash_pairs: list of commit/distro hash pairs
# promote_name: symlink name
hash_list = []
try:
for pair in request.json:
commit_hash = pair['commit_hash']
distro_hash = pair['distro_hash']
promote_name = pair['promote_name']
extended_hash = pair.get('extended_hash', None)
hash_item = [commit_hash, distro_hash, extended_hash, promote_name]
hash_list.append(hash_item)
except KeyError:
raise InvalidUsage('Missing parameters', status_code=400)
config_options = _get_config_options(app.config['CONFIG_FILE'])
session = _get_db()
# Now we will be running all checks for each combination
# Check for invalid promote names
for hash_item in hash_list:
commit_hash = hash_item[0]
distro_hash = hash_item[1]
extended_hash = hash_item[2]
promote_name = hash_item[3]
if (promote_name == 'consistent' or promote_name == 'current'):
raise InvalidUsage('Invalid promote_name %s for hash %s_%s' % (
promote_name, commit_hash, distro_hash),
status_code=403)
commit = _get_commit(session, commit_hash, distro_hash, extended_hash)
if commit is None:
raise InvalidUsage('commit_hash+distro_hash+extended_hash '
'combination not found for %s_%s_%s' % (
commit_hash, distro_hash, extended_hash),
status_code=404)
# If the commit has been purged, do not move on
if commit.flags & FLAG_PURGED:
raise InvalidUsage('commit_hash+distro_hash+extended_hash %s_%s_%s'
' has been purged, cannot promote it' % (
commit_hash, distro_hash, extended_hash),
status_code=410)
if config_options.use_components:
base_directory = os.path.join(app.config['REPO_PATH'],
"component/%s" % commit.component)
else:
base_directory = app.config['REPO_PATH']
target_link = os.path.join(base_directory, promote_name)
# Check for invalid target links, like ../promotename
target_dir = os.path.dirname(os.path.abspath(target_link))
if not os.path.samefile(target_dir, base_directory):
raise InvalidUsage('Invalid promote_name %s' % promote_name,
status_code=403)
# After all checks have been performed, do all promotions
rollback_list = []
for hash_item in hash_list:
rollback_item = {}
commit_hash = hash_item[0]
distro_hash = hash_item[1]
extended_hash = hash_item[2]
promote_name = hash_item[3]
commit = _get_commit(session, commit_hash, distro_hash, extended_hash)
# We should create a relative symlink
yumrepodir = commit.getshardedcommitdir()
if config_options.use_components:
base_directory = os.path.join(app.config['REPO_PATH'],
"component/%s" %
commit.component)
# In this case, the relative path should not include
# the component part
yumrepodir = yumrepodir.replace("component/%s/" % commit.component,
'')
else:
base_directory = app.config['REPO_PATH']
target_link = os.path.join(base_directory, promote_name)
rollback_item['target_link'] = target_link
rollback_item['previous_link'] = None
# Remove symlink if it exists, so we can create it again
if os.path.lexists(os.path.abspath(target_link)):
rollback_item['previous_link'] = os.readlink(
os.path.abspath(target_link))
os.remove(target_link)
rollback_list.append(rollback_item)
# This is the only destructive operation. If something fails here,
# we will try to roll everything back
try:
os.symlink(yumrepodir, target_link)
except Exception as e:
_rollback_batch_promotion(rollback_list)
raise InvalidUsage("Symlink creation failed with error: %s. "
"All previously created symlinks have been "
"rolled back." %
e, status_code=500)
timestamp = time.mktime(datetime.now().timetuple())
promotion = Promotion(commit_id=commit.id,
promotion_name=promote_name,
timestamp=timestamp, user=auth.username(),
component=commit.component,
aggregate_hash=None)
session.add(promotion)
# And finally, if we are using components, update the top-level
# repo file
repo_checksum = None
if config_options.use_components:
datadir = os.path.realpath(config_options.datadir)
repo_checksum = aggregate_repo_files(promote_name, datadir, session,
config_options.reponame,
hashed_dir=True)
promotion.aggregate_hash = repo_checksum
session.add(promotion)
# Close session and return the last promotion we did (which includes the
# repo checksum)
session.commit()
repo_hash = _repo_hash(commit)
repo_url = "%s/%s" % (config_options.baseurl, commit.getshardedcommitdir())
result = {'commit_hash': commit_hash,
'distro_hash': distro_hash,
'extended_hash': commit.extended_hash,
'repo_hash': repo_hash,
'repo_url': repo_url,
'promote_name': promote_name,
'component': commit.component,
'timestamp': timestamp,
'user': auth.username(),
'aggregate_hash': repo_checksum}
return jsonify(result), 201
def process_build_result_rpm(status,
packages,
session,
packages_to_process,
dev_mode=False,
run_cmd=False,
stop=False,
build_env=None,
head_only=False,
consistent=False,
failures=0):
config_options = getConfigOptions()
commit = status[0]
built_rpms = status[1]
notes = status[2]
exception = status[3]
commit_hash = commit.commit_hash
project = commit.project_name
project_info = session.query(Project).filter(
Project.project_name == project).first()
if not project_info:
project_info = Project(project_name=project, last_email=0)
exit_code = 0
if run_cmd:
if exception is not None:
exit_code = 1
if stop:
return exit_code
return exit_code
if exception is None:
commit.status = "SUCCESS"
commit.notes = notes
commit.artifacts = ",".join(built_rpms)
else:
logger.error("Received exception %s" % exception)
datadir = os.path.realpath(config_options.datadir)
yumrepodir = os.path.join(datadir, "repos",
commit.getshardedcommitdir())
logfile = os.path.join(yumrepodir, "rpmbuild.log")
# If the log file hasn't been created we add what we have
# This happens if the rpm build script didn't run.
if not os.path.exists(yumrepodir):
os.makedirs(yumrepodir)
if not os.path.exists(logfile):
with open(logfile, "w") as fp:
fp.write(str(exception))
if (isknownerror(logfile) and
(timesretried(project, session, commit_hash, commit.distro_hash) <
config_options.maxretries)):
logger.exception("Known error building packages for %s,"
" will retry later" % project)
commit.status = "RETRY"
commit.notes = str(exception)
# do not switch from an error exit code to a retry
# exit code
if exit_code != 1:
exit_code = 2
else:
exit_code = 1
if not project_info.suppress_email():
sendnotifymail(packages, commit)
project_info.sent_email()
session.add(project_info)
# allow to submit a gerrit review only if the last build
# was successful or non existent to avoid creating a gerrit
# review for the same problem multiple times.
if config_options.gerrit is not None:
if build_env:
env_vars = list(build_env)
else:
env_vars = []
last_build = getLastProcessedCommit(session, project)
if not last_build or last_build.status == 'SUCCESS':
try:
submit_review(commit, packages, env_vars)
except Exception:
logger.error('Unable to create review '
'see review.log')
else:
logger.info('Last build not successful '
'for %s' % project)
commit.status = "FAILED"
commit.notes = str(exception)
if stop:
return exit_code
# Add commit to the session
session.add(commit)
genreports(packages, head_only, session, packages_to_process)
# Export YAML file containing commit metadata
export_commit_yaml(commit)
try:
sync_repo(commit)
except Exception as e:
logger.error('Repo sync failed for project %s' % project)
consistent = False # If we were consistent before, we are not anymore
if exit_code == 0: # The commit was ok, so marking as failed
exit_code = 1
# We need to make the commit status be "failed"
commit.status = "FAILED"
commit.notes = str(e)
session.add(commit)
# And open a review if needed
if config_options.gerrit is not None:
if build_env:
env_vars = list(build_env)
else:
env_vars = []
try:
submit_review(commit, packages, env_vars)
except Exception:
logger.error('Unable to create review ' 'see review.log')
session.commit()
# Generate the current and consistent symlinks
if exception is None:
dirnames = ['current']
datadir = os.path.realpath(config_options.datadir)
yumrepodir = os.path.join(datadir, "repos",
commit.getshardedcommitdir())
yumrepodir_abs = os.path.join(datadir, yumrepodir)
if consistent:
dirnames.append('consistent')
else:
if config_options.use_components:
logger.info('%d packages not built correctly for component'
' %s: not updating the consistent symlink' %
(failures, commit.component))
else:
logger.info('%d packages not built correctly: not updating'
' the consistent symlink' % failures)
for dirname in dirnames:
if config_options.use_components:
target_repo_dir = os.path.join(datadir, "repos/component",
commit.component, dirname)
source_repo_dir = os.path.join(datadir, "repos/component",
commit.component)
else:
target_repo_dir = os.path.join(datadir, "repos", dirname)
source_repo_dir = os.path.join(datadir, "repos")
os.symlink(os.path.relpath(yumrepodir_abs, source_repo_dir),
target_repo_dir + "_")
os.rename(target_repo_dir + "_", target_repo_dir)
# If using components, synchronize the upper-level repo files
if config_options.use_components:
for dirname in dirnames:
aggregate_repo_files(dirname,
datadir,
session,
config_options.reponame,
hashed_dir=True)
# And synchronize them
sync_symlinks(commit)
if dev_mode is False:
if consistent:
# We have a consistent repo. Let's create a CIVote entry in the DB
vote = CIVote(commit_id=commit.id,
ci_name='consistent',
ci_url='',
ci_vote=True,
ci_in_progress=False,
timestamp=int(commit.dt_build),
notes='',
component=commit.component)
session.add(vote)
session.commit()
return exit_code