def add_bundles():
"""
Submit a request to add operator bundles to an index image.
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError('The input data must be a JSON object')
request = RequestAdd.from_json(payload)
db.session.add(request)
db.session.commit()
messaging.send_message_for_state_change(request, new_batch_msg=True)
overwrite_from_index = _should_force_overwrite() or payload.get(
'overwrite_from_index')
celery_queue = _get_user_queue(serial=overwrite_from_index)
args = _get_add_args(payload, request.id, overwrite_from_index,
celery_queue)
safe_args = _get_safe_args(args, payload)
error_callback = failed_request_callback.s(request.id)
try:
handle_add_request.apply_async(args=args,
link_error=error_callback,
argsrepr=repr(safe_args),
queue=celery_queue)
except kombu.exceptions.OperationalError:
handle_broker_error(request)
flask.current_app.logger.debug('Successfully scheduled request %d',
request.id)
return flask.jsonify(request.to_json()), 201
def test_create_empty_index_image_request(app, auth_env, client, db):
total_requests = 20
empty_index_revision = 'e16a8cd2e028'
# flask_login.current_user is used in RequestAdd.from_json and RequestRm.from_json,
# which requires a request context
with app.test_request_context(environ_base=auth_env):
# Generate some data to verify migration
data = {
'from_index': 'quay.io/namespace/index_image:latest',
'binary_image': 'quay.io/namespace/binary_image:latest',
}
request = RequestCreateEmptyIndex.from_json(data)
db.session.add(request)
for i in range(total_requests):
request_class = random.choice(
(RequestAdd, RequestRm, RequestCreateEmptyIndex))
if request_class == RequestAdd:
data = {
'binary_image': 'quay.io/namespace/binary_image:latest',
'bundles': [f'quay.io/namespace/bundle:{i}'],
'from_index': f'quay.io/namespace/repo:{i}',
}
request = RequestAdd.from_json(data)
elif request_class == RequestRm:
data = {
'binary_image': 'quay.io/namespace/binary_image:latest',
'operators': [f'operator-{i}'],
'from_index': f'quay.io/namespace/repo:{i}',
}
request = RequestRm.from_json(data)
elif request_class == RequestCreateEmptyIndex:
data = {
'from_index': f'quay.io/namespace/index_image:{i}',
'binary_image': 'quay.io/namespace/binary_image:latest',
}
request = RequestCreateEmptyIndex.from_json(data)
if i % 5 == 0:
# Simulate failed request
request.add_state('failed', 'Failed due to an unknown error')
db.session.add(request)
db.session.commit()
expected_rv_json = client.get(
f'/api/v1/builds?per_page={total_requests}&verbose=true').json
flask_migrate.downgrade(revision=empty_index_revision)
flask_migrate.upgrade()
actual_rv_json = client.get(
f'/api/v1/builds?per_page={total_requests}&verbose=true').json
assert expected_rv_json == actual_rv_json
def add_bundles():
"""
Submit a request to add operator bundles to an index image.
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError('The input data must be a JSON object')
request = RequestAdd.from_json(payload)
db.session.add(request)
db.session.commit()
messaging.send_message_for_state_change(request, new_batch_msg=True)
overwrite_from_index = _should_force_overwrite() or payload.get(
'overwrite_from_index')
celery_queue = _get_user_queue(serial=overwrite_from_index)
args = [
payload['bundles'],
payload['binary_image'],
request.id,
payload.get('from_index'),
payload.get('add_arches'),
payload.get('cnr_token'),
payload.get('organization'),
payload.get('force_backport'),
overwrite_from_index,
payload.get('overwrite_from_index_token'),
flask.current_app.config['IIB_GREENWAVE_CONFIG'].get(celery_queue),
]
safe_args = copy.copy(args)
if payload.get('cnr_token'):
safe_args[safe_args.index(payload['cnr_token'])] = '*****'
if payload.get('overwrite_from_index_token'):
safe_args[safe_args.index(
payload['overwrite_from_index_token'])] = '*****'
error_callback = failed_request_callback.s(request.id)
try:
handle_add_request.apply_async(args=args,
link_error=error_callback,
argsrepr=repr(safe_args),
queue=celery_queue)
except kombu.exceptions.OperationalError:
handle_broker_error(request)
flask.current_app.logger.debug('Successfully scheduled request %d',
request.id)
return flask.jsonify(request.to_json()), 201
def test_abort_when_downgrading_from_regenerate_bundle_request(
app, auth_env, client, db):
"""Verify downgrade is prevented if "regenerate-bundle" requests exist."""
total_requests = 20
# flask_login.current_user is used in Request*.from_json which requires a request context
with app.test_request_context(environ_base=auth_env):
# Always add a RequestRegenerateBundle to ensure sufficient test data is available
data = {'from_bundle_image': 'quay.io/namespace/bundle-image:latest'}
request = RequestRegenerateBundle.from_json(data)
db.session.add(request)
# One request was already added, let's add the remaining ones
for i in range(total_requests - 1):
request_class = random.choice(
(RequestAdd, RequestRm, RequestRegenerateBundle))
if request_class == RequestAdd:
data = {
'binary_image': 'quay.io/namespace/binary_image:latest',
'bundles': [f'quay.io/namespace/bundle:{i}'],
'from_index': f'quay.io/namespace/repo:{i}',
}
request = RequestAdd.from_json(data)
elif request_class == RequestRm:
data = {
'binary_image': 'quay.io/namespace/binary_image:latest',
'operators': [f'operator-{i}'],
'from_index': f'quay.io/namespace/repo:{i}',
}
request = RequestRm.from_json(data)
else:
data = {
'from_bundle_image':
'quay.io/namespace/bundle-image:latest'
}
request = RequestRegenerateBundle.from_json(data)
db.session.add(request)
db.session.add(request)
db.session.commit()
# flask_migrate raises a SystemExit exception regardless of what's raised from the
# downgrade function. This exception doesn't hold a reference to the RuntimeError
# we expect from the downgrade function in the migration script. The best we can
# do is catch the SystemExit exception.
with pytest.raises(SystemExit):
flask_migrate.downgrade(revision=INITIAL_DB_REVISION)
def test_get_builds(app, auth_env, client, db):
total_requests = 50
# flask_login.current_user is used in RequestAdd.from_json, which requires a request context
with app.test_request_context(environ_base=auth_env):
for i in range(total_requests):
data = {
'binary_image': 'quay.io/namespace/binary_image:latest',
'bundles': [f'quay.io/namespace/bundle:{i}'],
'from_index': f'quay.io/namespace/repo:{i}',
}
request = RequestAdd.from_json(data)
if i % 5 == 0:
request.add_state('failed', 'Failed due to an unknown error')
db.session.add(request)
db.session.commit()
rv_json = client.get('/api/v1/builds?page=2').json
# Verify the order_by is correct
assert rv_json['items'][0][
'id'] == total_requests - app.config['IIB_MAX_PER_PAGE']
assert len(rv_json['items']) == app.config['IIB_MAX_PER_PAGE']
# This key is only present the verbose=true
assert 'state_history' not in rv_json['items'][0]
assert rv_json['meta']['page'] == 2
assert rv_json['meta']['pages'] == 3
assert rv_json['meta']['per_page'] == app.config['IIB_MAX_PER_PAGE']
assert rv_json['meta']['total'] == total_requests
rv_json = client.get('/api/v1/builds?state=failed&per_page=5').json
total_failed_requests = total_requests // 5
assert len(rv_json['items']) == 5
assert 'state=failed' in rv_json['meta']['next']
assert rv_json['meta']['page'] == 1
assert rv_json['meta']['pages'] == 2
assert rv_json['meta']['per_page'] == 5
assert rv_json['meta']['total'] == total_failed_requests
rv_json = client.get('/api/v1/builds?batch=3').json
assert len(rv_json['items']) == 1
assert 'batch=3' in rv_json['meta']['first']
assert rv_json['meta']['total'] == 1
rv_json = client.get('/api/v1/builds?verbose=true&per_page=1').json
# This key is only present the verbose=true
assert 'state_history' in rv_json['items'][0]
def test_migrate_to_polymorphic_requests(app, auth_env, client, db):
total_requests = 20
# flask_login.current_user is used in RequestAdd.from_json and RequestRm.from_json,
# which requires a request context
with app.test_request_context(environ_base=auth_env):
# Generate some data to verify migration
for i in range(total_requests):
if random.choice((True, False)):
data = {
'binary_image': 'quay.io/namespace/binary_image:latest',
'bundles': [f'quay.io/namespace/bundle:{i}'],
'from_index': f'quay.io/namespace/repo:{i}',
}
request = RequestAdd.from_json(data)
else:
data = {
'binary_image': 'quay.io/namespace/binary_image:latest',
'operators': [f'operator-{i}'],
'from_index': f'quay.io/namespace/repo:{i}',
}
request = RequestRm.from_json(data)
if i % 5 == 0:
# Simulate failed request
request.add_state('failed', 'Failed due to an unknown error')
db.session.add(request)
db.session.commit()
expected_rv_json = client.get(
f'/api/v1/builds?per_page={total_requests}&verbose=true').json
flask_migrate.downgrade(revision=INITIAL_DB_REVISION)
flask_migrate.upgrade()
actual_rv_json = client.get(
f'/api/v1/builds?per_page={total_requests}&verbose=true').json
assert expected_rv_json == actual_rv_json
def add_rm_batch():
"""
Submit a batch of requests to add or remove operators from an index image.
Note: Any duplicate bundle will be removed from payload when adding operators.
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
Batch.validate_batch_request_params(payload)
batch = Batch(annotations=payload.get('annotations'))
db.session.add(batch)
requests = []
# Iterate through all the build requests and verify that the requests are valid before
# committing them and scheduling the tasks
for build_request in payload['build_requests']:
try:
if build_request.get('operators'):
# Check for the validity of a RM request
request = RequestRm.from_json(build_request, batch)
elif build_request.get('bundles'):
build_request_uniq = copy.deepcopy(build_request)
build_request_uniq['bundles'] = _get_unique_bundles(build_request_uniq['bundles'])
# Check for the validity of an Add request
request = RequestAdd.from_json(build_request_uniq, batch)
else:
raise ValidationError('Build request is not a valid Add/Rm request.')
except ValidationError as e:
raise ValidationError(
f'{str(e).rstrip(".")}. This occurred on the build request in '
f'index {payload["build_requests"].index(build_request)}.'
)
db.session.add(request)
requests.append(request)
db.session.commit()
messaging.send_messages_for_new_batch_of_requests(requests)
request_jsons = []
# This list will be used for the log message below and avoids the need of having to iterate
# through the list of requests another time
processed_request_ids = []
for build_request, request in zip(payload['build_requests'], requests):
request_jsons.append(request.to_json())
overwrite_from_index = build_request.get('overwrite_from_index')
celery_queue = _get_user_queue(serial=overwrite_from_index)
if isinstance(request, RequestAdd):
args = _get_add_args(build_request, request, overwrite_from_index, celery_queue)
elif isinstance(request, RequestRm):
args = _get_rm_args(build_request, request, overwrite_from_index)
safe_args = _get_safe_args(args, build_request)
error_callback = failed_request_callback.s(request.id)
try:
if isinstance(request, RequestAdd):
handle_add_request.apply_async(
args=args,
link_error=error_callback,
argsrepr=repr(safe_args),
queue=celery_queue,
)
else:
handle_rm_request.apply_async(
args=args,
link_error=error_callback,
argsrepr=repr(safe_args),
queue=celery_queue,
)
except kombu.exceptions.OperationalError:
unprocessed_requests = [r for r in requests if str(r.id) not in processed_request_ids]
handle_broker_batch_error(unprocessed_requests)
processed_request_ids.append(str(request.id))
flask.current_app.logger.debug(
'Successfully scheduled the batch %d with requests: %s',
batch.id,
', '.join(processed_request_ids),
)
return flask.jsonify(request_jsons), 201
def test_get_build(app, auth_env, client, db):
# flask_login.current_user is used in RequestAdd.from_json, which requires a request context
with app.test_request_context(environ_base=auth_env):
data = {
'binary_image': 'quay.io/namespace/binary_image:latest',
'bundles': [f'quay.io/namespace/bundle:1.0-3'],
'from_index': f'quay.io/namespace/repo:latest',
}
request = RequestAdd.from_json(data)
request.binary_image_resolved = Image.get_or_create(
'quay.io/namespace/binary_image@sha256:abcdef')
request.from_index_resolved = Image.get_or_create(
'quay.io/namespace/from_index@sha256:defghi')
request.index_image = Image.get_or_create(
'quay.io/namespace/index@sha256:fghijk')
request.add_architecture('amd64')
request.add_architecture('s390x')
request.add_state('complete', 'Completed successfully')
db.session.add(request)
db.session.commit()
rv = client.get('/api/v1/builds/1').json
for state in rv['state_history']:
# Set this to a stable timestamp so the tests are dependent on it
state['updated'] = '2020-02-12T17:03:00Z'
rv['updated'] = '2020-02-12T17:03:00Z'
expected = {
'arches': ['amd64', 's390x'],
'batch':
1,
'batch_annotations':
None,
'binary_image':
'quay.io/namespace/binary_image:latest',
'binary_image_resolved':
'quay.io/namespace/binary_image@sha256:abcdef',
'bundle_mapping': {},
'bundles': ['quay.io/namespace/bundle:1.0-3'],
'from_index':
'quay.io/namespace/repo:latest',
'from_index_resolved':
'quay.io/namespace/from_index@sha256:defghi',
'id':
1,
'index_image':
'quay.io/namespace/index@sha256:fghijk',
'organization':
None,
'removed_operators': [],
'request_type':
'add',
'state':
'complete',
'state_history': [
{
'state': 'complete',
'state_reason': 'Completed successfully',
'updated': '2020-02-12T17:03:00Z',
},
{
'state': 'in_progress',
'state_reason': 'The request was initiated',
'updated': '2020-02-12T17:03:00Z',
},
],
'state_reason':
'Completed successfully',
'updated':
'2020-02-12T17:03:00Z',
'user':
'******',
}
assert rv == expected