def report_result():
# job_id: name of CI
# commit_hash: commit hash
# distro_hash: distro hash
# url: URL where more information can be found
# timestamp: CI execution timestamp
# success: boolean
# notes(optional): notes
if request.headers['Content-Type'] != 'application/json':
raise InvalidUsage('Unsupported Media Type, use JSON', status_code=415)
try:
commit_hash = request.json['commit_hash']
distro_hash = request.json['distro_hash']
timestamp = request.json['timestamp']
job_id = request.json['job_id']
success = request.json['success']
url = request.json['url']
except KeyError:
raise InvalidUsage('Missing parameters', status_code=400)
notes = request.json.get('notes', '')
session = getSession(app.config['DB_PATH'])
commit = session.query(Commit).filter(
Commit.status == 'SUCCESS', Commit.commit_hash == commit_hash,
Commit.distro_hash == distro_hash).first()
if commit is None:
raise InvalidUsage('commit_hash+distro_hash combination not found',
status_code=404)
commit_id = commit.id
vote = CIVote(commit_id=commit_id,
ci_name=job_id,
ci_url=url,
ci_vote=bool(strtobool(success)),
ci_in_progress=False,
timestamp=int(timestamp),
notes=notes,
user=auth.username())
session.add(vote)
session.commit()
result = {
'commit_hash': commit_hash,
'distro_hash': distro_hash,
'timestamp': timestamp,
'job_id': job_id,
'success': bool(strtobool(success)),
'in_progress': False,
'url': url,
'notes': notes,
'user': auth.username()
}
closeSession(session)
return jsonify(result), 201
def report_result():
# job_id: name of CI
# commit_hash: commit hash
# distro_hash: distro hash
# url: URL where more information can be found
# timestamp: CI execution timestamp
# success: boolean
# notes(optional): notes
try:
commit_hash = request.json['commit_hash']
distro_hash = request.json['distro_hash']
timestamp = request.json['timestamp']
job_id = request.json['job_id']
success = request.json['success']
url = request.json['url']
except KeyError:
raise InvalidUsage('Missing parameters', status_code=400)
notes = request.json.get('notes', '')
session = getSession(app.config['DB_PATH'])
commit = _get_commit(session, commit_hash, distro_hash)
if commit is None:
raise InvalidUsage('commit_hash+distro_hash combination not found',
status_code=404)
commit_id = commit.id
vote = CIVote(commit_id=commit_id, ci_name=job_id, ci_url=url,
ci_vote=bool(strtobool(success)), ci_in_progress=False,
timestamp=int(timestamp), notes=notes,
user=auth.username())
session.add(vote)
session.commit()
result = {'commit_hash': commit_hash,
'distro_hash': distro_hash,
'timestamp': timestamp,
'job_id': job_id,
'success': bool(strtobool(success)),
'in_progress': False,
'url': url,
'notes': notes,
'user': auth.username()}
closeSession(session)
return jsonify(result), 201
def promote_batch():
# hash_pairs: list of commit/distro hash pairs
# promote_name: symlink name
hash_list = []
try:
for pair in request.json:
commit_hash = pair['commit_hash']
distro_hash = pair['distro_hash']
promote_name = pair['promote_name']
extended_hash = pair.get('extended_hash', None)
hash_item = [commit_hash, distro_hash, extended_hash, promote_name]
hash_list.append(hash_item)
except KeyError:
raise InvalidUsage('Missing parameters', status_code=400)
config_options = _get_config_options(app.config['CONFIG_FILE'])
session = _get_db()
# Now we will be running all checks for each combination
# Check for invalid promote names
for hash_item in hash_list:
commit_hash = hash_item[0]
distro_hash = hash_item[1]
extended_hash = hash_item[2]
promote_name = hash_item[3]
if (promote_name == 'consistent' or promote_name == 'current'):
raise InvalidUsage('Invalid promote_name %s for hash %s_%s' % (
promote_name, commit_hash, distro_hash),
status_code=403)
commit = _get_commit(session, commit_hash, distro_hash, extended_hash)
if commit is None:
raise InvalidUsage('commit_hash+distro_hash+extended_hash '
'combination not found for %s_%s_%s' % (
commit_hash, distro_hash, extended_hash),
status_code=404)
# If the commit has been purged, do not move on
if commit.flags & FLAG_PURGED:
raise InvalidUsage('commit_hash+distro_hash+extended_hash %s_%s_%s'
' has been purged, cannot promote it' % (
commit_hash, distro_hash, extended_hash),
status_code=410)
if config_options.use_components:
base_directory = os.path.join(app.config['REPO_PATH'],
"component/%s" % commit.component)
else:
base_directory = app.config['REPO_PATH']
target_link = os.path.join(base_directory, promote_name)
# Check for invalid target links, like ../promotename
target_dir = os.path.dirname(os.path.abspath(target_link))
if not os.path.samefile(target_dir, base_directory):
raise InvalidUsage('Invalid promote_name %s' % promote_name,
status_code=403)
# After all checks have been performed, do all promotions
rollback_list = []
for hash_item in hash_list:
rollback_item = {}
commit_hash = hash_item[0]
distro_hash = hash_item[1]
extended_hash = hash_item[2]
promote_name = hash_item[3]
commit = _get_commit(session, commit_hash, distro_hash, extended_hash)
# We should create a relative symlink
yumrepodir = commit.getshardedcommitdir()
if config_options.use_components:
base_directory = os.path.join(app.config['REPO_PATH'],
"component/%s" %
commit.component)
# In this case, the relative path should not include
# the component part
yumrepodir = yumrepodir.replace("component/%s/" % commit.component,
'')
else:
base_directory = app.config['REPO_PATH']
target_link = os.path.join(base_directory, promote_name)
rollback_item['target_link'] = target_link
rollback_item['previous_link'] = None
# Remove symlink if it exists, so we can create it again
if os.path.lexists(os.path.abspath(target_link)):
rollback_item['previous_link'] = os.readlink(
os.path.abspath(target_link))
os.remove(target_link)
rollback_list.append(rollback_item)
# This is the only destructive operation. If something fails here,
# we will try to roll everything back
try:
os.symlink(yumrepodir, target_link)
except Exception as e:
_rollback_batch_promotion(rollback_list)
raise InvalidUsage("Symlink creation failed with error: %s. "
"All previously created symlinks have been "
"rolled back." %
e, status_code=500)
timestamp = time.mktime(datetime.now().timetuple())
promotion = Promotion(commit_id=commit.id,
promotion_name=promote_name,
timestamp=timestamp, user=auth.username(),
component=commit.component,
aggregate_hash=None)
session.add(promotion)
# And finally, if we are using components, update the top-level
# repo file
repo_checksum = None
if config_options.use_components:
datadir = os.path.realpath(config_options.datadir)
repo_checksum = aggregate_repo_files(promote_name, datadir, session,
config_options.reponame,
hashed_dir=True)
promotion.aggregate_hash = repo_checksum
session.add(promotion)
# Close session and return the last promotion we did (which includes the
# repo checksum)
session.commit()
repo_hash = _repo_hash(commit)
repo_url = "%s/%s" % (config_options.baseurl, commit.getshardedcommitdir())
result = {'commit_hash': commit_hash,
'distro_hash': distro_hash,
'extended_hash': commit.extended_hash,
'repo_hash': repo_hash,
'repo_url': repo_url,
'promote_name': promote_name,
'component': commit.component,
'timestamp': timestamp,
'user': auth.username(),
'aggregate_hash': repo_checksum}
return jsonify(result), 201
def promote():
# commit_hash: commit hash
# distro_hash: distro hash
# extended_hash (optional): extended hash
# promote_name: symlink name
try:
commit_hash = request.json['commit_hash']
distro_hash = request.json['distro_hash']
promote_name = request.json['promote_name']
except KeyError:
raise InvalidUsage('Missing parameters', status_code=400)
extended_hash = request.json.get('extended_hash', None)
# Check for invalid promote names
if (promote_name == 'consistent' or promote_name == 'current'):
raise InvalidUsage('Invalid promote_name %s' % promote_name,
status_code=403)
config_options = _get_config_options(app.config['CONFIG_FILE'])
session = _get_db()
commit = _get_commit(session, commit_hash, distro_hash, extended_hash)
if commit is None:
raise InvalidUsage('commit_hash+distro_hash+extended_hash combination'
' not found', status_code=404)
# If the commit has been purged, do not move on
if commit.flags & FLAG_PURGED:
raise InvalidUsage('commit_hash+distro_hash+extended_hash has been '
'purged, cannot promote it', status_code=410)
if config_options.use_components:
base_directory = os.path.join(app.config['REPO_PATH'], "component/%s" %
commit.component)
else:
base_directory = app.config['REPO_PATH']
target_link = os.path.join(base_directory, promote_name)
# Check for invalid target links, like ../promotename
target_dir = os.path.dirname(os.path.abspath(target_link))
if not os.path.samefile(target_dir, base_directory):
raise InvalidUsage('Invalid promote_name %s' % promote_name,
status_code=403)
# We should create a relative symlink
yumrepodir = commit.getshardedcommitdir()
if config_options.use_components:
# In this case, the relative path should not include
# the component part
yumrepodir = yumrepodir.replace("component/%s/" % commit.component, '')
# Remove symlink if it exists, so we can create it again
if os.path.lexists(os.path.abspath(target_link)):
os.remove(target_link)
try:
os.symlink(yumrepodir, target_link)
except Exception as e:
raise InvalidUsage("Symlink creation failed with error: %s" %
e, status_code=500)
# Once the updated symlink is created, if we are using components
# we need to update the top-level repo file
repo_checksum = None
if config_options.use_components:
datadir = os.path.realpath(config_options.datadir)
repo_checksum = aggregate_repo_files(promote_name, datadir, session,
config_options.reponame,
hashed_dir=True)
timestamp = time.mktime(datetime.now().timetuple())
promotion = Promotion(commit_id=commit.id, promotion_name=promote_name,
timestamp=timestamp, user=auth.username(),
component=commit.component,
aggregate_hash=repo_checksum)
session.add(promotion)
session.commit()
repo_hash = _repo_hash(commit)
repo_url = "%s/%s" % (config_options.baseurl, commit.getshardedcommitdir())
result = {'commit_hash': commit_hash,
'distro_hash': distro_hash,
'extended_hash': commit.extended_hash,
'repo_hash': repo_hash,
'repo_url': repo_url,
'promote_name': promote_name,
'component': commit.component,
'timestamp': timestamp,
'user': auth.username(),
'aggregate_hash': repo_checksum}
return jsonify(result), 201
def report_result():
# job_id: name of CI
# commit_hash: commit hash
# distro_hash: distro hash
# extended_hash(optional): extended hash
# aggregate_hash: hash of aggregate.
# url: URL where more information can be found
# timestamp: CI execution timestamp
# success: boolean
# notes(optional): notes
# Either commit_hash+distro_hash or aggregate_hash must be provided
try:
timestamp = request.json['timestamp']
job_id = request.json['job_id']
success = request.json['success']
url = request.json['url']
except KeyError:
raise InvalidUsage('Missing parameters', status_code=400)
commit_hash = request.json.get('commit_hash', None)
distro_hash = request.json.get('distro_hash', None)
extended_hash = request.json.get('extended_hash', None)
aggregate_hash = request.json.get('aggregate_hash', None)
if not commit_hash and not distro_hash and not aggregate_hash:
raise InvalidUsage('Missing parameters', status_code=400)
if commit_hash and not distro_hash:
raise InvalidUsage('If commit_hash is provided, distro_hash '
'must be provided too', status_code=400)
if distro_hash and not commit_hash:
raise InvalidUsage('If distro_hash is provided, commit_hash '
'must be provided too', status_code=400)
if (aggregate_hash and distro_hash) or (aggregate_hash and commit_hash):
raise InvalidUsage('aggregate_hash and commit/distro_hash cannot be '
'combined', status_code=400)
notes = request.json.get('notes', '')
session = _get_db()
# We have two paths here: one for votes on commit/distro/extended hash,
# another for votes on aggregate_hash
component = None
if commit_hash:
commit = _get_commit(session, commit_hash, distro_hash, extended_hash)
if commit is None:
raise InvalidUsage('commit_hash+distro_hash+extended_hash '
'combination not found', status_code=404)
commit_id = commit.id
out_ext_hash = commit.extended_hash
component = commit.component
vote = CIVote(commit_id=commit_id, ci_name=job_id, ci_url=url,
ci_vote=bool(strtobool(success)), ci_in_progress=False,
timestamp=int(timestamp), notes=notes,
user=auth.username(), component=component)
else:
out_ext_hash = None
prom = session.query(Promotion).filter(
Promotion.aggregate_hash == aggregate_hash).first()
if prom is None:
raise InvalidUsage('aggregate_hash not found',
status_code=400)
vote = CIVote_Aggregate(ref_hash=aggregate_hash, ci_name=job_id,
ci_url=url, ci_vote=bool(strtobool(success)),
ci_in_progress=False, timestamp=int(timestamp),
notes=notes, user=auth.username())
session.add(vote)
session.commit()
result = {'commit_hash': commit_hash,
'distro_hash': distro_hash,
'extended_hash': out_ext_hash,
'aggregate_hash': aggregate_hash,
'timestamp': timestamp,
'job_id': job_id,
'success': bool(strtobool(success)),
'in_progress': False,
'url': url,
'notes': notes,
'user': auth.username(),
'component': component}
return jsonify(result), 201
def last_tested_repo_POST():
# max_age: Maximum age in hours, used as base for the search
# success(optional): find repos with a successful/unsuccessful vote
# job_id(optional); name of the CI that sent the vote
# reporting_job_id: name of the CI that will test this repo
# sequential_mode(optional): if set to true, change the search algorithm
# to only use previous_job_id as CI name to
# search for. Defaults to false
# previous_job_id(optional): CI name to search for, if sequential_mode is
# True
# component(optional): only get votes for this component
max_age = request.json.get('max_age', None)
my_job_id = request.json.get('reporting_job_id', None)
job_id = request.json.get('job_id', None)
success = request.json.get('success', None)
sequential_mode = request.json.get('sequential_mode', None)
previous_job_id = request.json.get('previous_job_id', None)
component = request.json.get('component', None)
if success is not None:
success = bool(strtobool(success))
if sequential_mode is not None:
sequential_mode = bool(strtobool(sequential_mode))
if sequential_mode and previous_job_id is None:
raise InvalidUsage('Missing parameter previous_job_id',
status_code=400)
if (max_age is None or my_job_id is None):
raise InvalidUsage('Missing parameters', status_code=400)
# Calculate timestamp as now - max_age
if int(max_age) == 0:
timestamp = 0
else:
oldest_time = datetime.now() - timedelta(hours=int(max_age))
timestamp = time.mktime(oldest_time.timetuple())
session = _get_db()
try:
if sequential_mode:
# CI pipeline case
vote = getVote(session, timestamp, success, previous_job_id,
component=component, fallback=False)
else:
# Normal case
vote = getVote(session, timestamp, success, job_id,
component=component)
except Exception as e:
raise e
newvote = CIVote(commit_id=vote.commit_id, ci_name=my_job_id,
ci_url='', ci_vote=False, ci_in_progress=True,
timestamp=int(time.time()), notes='',
user=auth.username(), component=vote.component)
session.add(newvote)
session.commit()
commit = session.query(Commit).filter(
Commit.status == 'SUCCESS',
Commit.id == vote.commit_id).first()
result = {'commit_hash': commit.commit_hash,
'distro_hash': commit.distro_hash,
'extended_hash': commit.extended_hash,
'timestamp': newvote.timestamp,
'job_id': newvote.ci_name,
'success': newvote.ci_vote,
'in_progress': newvote.ci_in_progress,
'user': newvote.user,
'component': newvote.component}
return jsonify(result), 201
def promote():
# commit_hash: commit hash
# distro_hash: distro hash
# promote_name: symlink name
try:
commit_hash = request.json['commit_hash']
distro_hash = request.json['distro_hash']
promote_name = request.json['promote_name']
except KeyError:
raise InvalidUsage('Missing parameters', status_code=400)
# Check for invalid promote names
if (promote_name == 'consistent' or promote_name == 'current'):
raise InvalidUsage('Invalid promote_name %s' % promote_name,
status_code=403)
config_options = _get_config_options(app.config['CONFIG_FILE'])
session = getSession(app.config['DB_PATH'])
commit = _get_commit(session, commit_hash, distro_hash)
if commit is None:
raise InvalidUsage('commit_hash+distro_hash combination not found',
status_code=404)
# If the commit has been purged, do not move on
if commit.flags & FLAG_PURGED:
raise InvalidUsage('commit_hash+distro_hash has been purged, cannot '
'promote it', status_code=410)
target_link = os.path.join(app.config['REPO_PATH'], promote_name)
# Check for invalid target links, like ../promotename
target_dir = os.path.dirname(os.path.abspath(target_link))
if not os.path.samefile(target_dir, app.config['REPO_PATH']):
raise InvalidUsage('Invalid promote_name %s' % promote_name,
status_code=403)
# We should create a relative symlink
yumrepodir = commit.getshardedcommitdir()
# Remove symlink if it exists, so we can create it again
if os.path.lexists(os.path.abspath(target_link)):
os.remove(target_link)
try:
os.symlink(yumrepodir, target_link)
except Exception as e:
raise InvalidUsage("Symlink creation failed with error: %s" %
e, status_code=500)
timestamp = time.mktime(datetime.now().timetuple())
promotion = Promotion(commit_id=commit.id, promotion_name=promote_name,
timestamp=timestamp, user=auth.username())
session.add(promotion)
session.commit()
repo_hash = _repo_hash(commit)
repo_url = "%s/%s" % (config_options.baseurl, yumrepodir)
result = {'commit_hash': commit_hash,
'distro_hash': distro_hash,
'repo_hash': repo_hash,
'repo_url': repo_url,
'promote_name': promote_name,
'timestamp': timestamp,
'user': auth.username()}
closeSession(session)
return jsonify(result), 201
def last_tested_repo_POST():
# max_age: Maximum age in hours, used as base for the search
# success(optional): find repos with a successful/unsuccessful vote
# job_id(optional); name of the CI that sent the vote
# reporting_job_id: name of the CI that will test this repo
# sequential_mode(optional): if set to true, change the search algorithm
# to only use previous_job_id as CI name to
# search for. Defaults to false
# previous_job_id(optional): CI name to search for, if sequential_mode is
# True
max_age = request.json.get('max_age', None)
my_job_id = request.json.get('reporting_job_id', None)
job_id = request.json.get('job_id', None)
success = request.json.get('success', None)
sequential_mode = request.json.get('sequential_mode', None)
previous_job_id = request.json.get('previous_job_id', None)
if success is not None:
success = bool(strtobool(success))
if sequential_mode is not None:
sequential_mode = bool(strtobool(sequential_mode))
if sequential_mode and previous_job_id is None:
raise InvalidUsage('Missing parameter previous_job_id',
status_code=400)
if (max_age is None or my_job_id is None):
raise InvalidUsage('Missing parameters', status_code=400)
# Calculate timestamp as now - max_age
if int(max_age) == 0:
timestamp = 0
else:
oldest_time = datetime.now() - timedelta(hours=int(max_age))
timestamp = time.mktime(oldest_time.timetuple())
session = getSession(app.config['DB_PATH'])
try:
if sequential_mode:
# CI pipeline case
vote = getVote(session, timestamp, success, previous_job_id,
fallback=False)
else:
# Normal case
vote = getVote(session, timestamp, success, job_id)
except Exception as e:
raise e
newvote = CIVote(commit_id=vote.commit_id, ci_name=my_job_id,
ci_url='', ci_vote=False, ci_in_progress=True,
timestamp=int(time.time()), notes='',
user=auth.username())
session.add(newvote)
session.commit()
commit = session.query(Commit).filter(
Commit.status == 'SUCCESS',
Commit.id == vote.commit_id).first()
result = {'commit_hash': commit.commit_hash,
'distro_hash': commit.distro_hash,
'timestamp': newvote.timestamp,
'job_id': newvote.ci_name,
'success': newvote.ci_vote,
'in_progress': newvote.ci_in_progress,
'user': newvote.user}
closeSession(session)
return jsonify(result), 201
def promote():
# commit_hash: commit hash
# distro_hash: distro hash
# promote_name: symlink name
if request.headers['Content-Type'] != 'application/json':
raise InvalidUsage('Unsupported Media Type, use JSON', status_code=415)
try:
commit_hash = request.json['commit_hash']
distro_hash = request.json['distro_hash']
promote_name = request.json['promote_name']
except KeyError:
raise InvalidUsage('Missing parameters', status_code=400)
# Check for invalid promote names
if (promote_name == 'consistent' or promote_name == 'current'):
raise InvalidUsage('Invalid promote_name %s' % promote_name,
status_code=403)
config_options = _get_config_options(app.config['CONFIG_FILE'])
session = getSession(app.config['DB_PATH'])
commit = session.query(Commit).filter(
Commit.status == 'SUCCESS', Commit.commit_hash == commit_hash,
Commit.distro_hash == distro_hash).first()
if commit is None:
raise InvalidUsage('commit_hash+distro_hash combination not found',
status_code=404)
target_link = os.path.join(app.config['REPO_PATH'], promote_name)
# Check for invalid target links, like ../promotename
target_dir = os.path.dirname(os.path.abspath(target_link))
if not os.path.samefile(target_dir, app.config['REPO_PATH']):
raise InvalidUsage('Invalid promote_name %s' % promote_name,
status_code=403)
# We should create a relative symlink
yumrepodir = commit.getshardedcommitdir()
# Remove symlink if it exists, so we can create it again
if os.path.lexists(os.path.abspath(target_link)):
os.remove(target_link)
try:
os.symlink(yumrepodir, target_link)
except Exception as e:
raise InvalidUsage("Symlink creation failed with error: %s" % e,
status_code=500)
timestamp = time.mktime(datetime.now().timetuple())
promotion = Promotion(commit_id=commit.id,
promotion_name=promote_name,
timestamp=timestamp,
user=auth.username())
session.add(promotion)
session.commit()
repo_hash = _repo_hash(commit)
repo_url = "%s/%s" % (config_options.baseurl, yumrepodir)
result = {
'commit_hash': commit_hash,
'distro_hash': distro_hash,
'repo_hash': repo_hash,
'repo_url': repo_url,
'promote_name': promote_name,
'timestamp': timestamp,
'user': auth.username()
}
closeSession(session)
return jsonify(result), 201