def test_create_request_filter_state(app, auth_env, client, db):
"""Test that requests can be filtered by state."""
repo_template = 'https://github.com/release-engineering/retrodep{}.git'
# flask_login.current_user is used in Request.from_json, which requires a request context
with app.test_request_context(environ_base=auth_env):
# Make a request in 'in_progress' state
data = {
'repo': repo_template.format(0),
'ref': 'c50b93a32df1c9d700e3e80996845bc2e13be848',
'pkg_managers': ['gomod'],
}
request = Request.from_json(data)
db.session.add(request)
# Make a request in 'complete' state
data_complete = {
'repo': repo_template.format(1),
'ref': 'e1be527f39ec31323f0454f7d1422c6260b00580',
'pkg_managers': ['gomod'],
}
request_complete = Request.from_json(data_complete)
request_complete.add_state('complete', 'Completed successfully')
db.session.add(request_complete)
db.session.commit()
for state in ('in_progress', 'complete'):
rv = client.get(f'/api/v1/requests?state={state}')
assert rv.status_code == 200
fetched_requests = rv.json['items']
assert len(fetched_requests) == 1
assert fetched_requests[0]['state'] == state
def test_fetch_paginated_requests(mock_chain, app, auth_env, client, db):
repo_template = 'https://github.com/release-engineering/retrodep{}.git'
# flask_login.current_user is used in Request.from_json, which requires a request context
with app.test_request_context(environ_base=auth_env):
for i in range(50):
data = {
'repo': repo_template.format(i),
'ref': 'c50b93a32df1c9d700e3e80996845bc2e13be848',
'pkg_managers': ['gomod'],
}
request = Request.from_json(data)
db.session.add(request)
db.session.commit()
# Sane defaults are provided
rv = client.get('/api/v1/requests')
assert rv.status_code == 200
response = json.loads(rv.data.decode('utf-8'))
fetched_requests = response['items']
assert len(fetched_requests) == 20
for repo_number, request in enumerate(fetched_requests):
assert request['repo'] == repo_template.format(repo_number)
# per_page and page parameters are honored
rv = client.get('/api/v1/requests?page=2&per_page=10')
assert rv.status_code == 200
response = json.loads(rv.data.decode('utf-8'))
fetched_requests = response['items']
assert len(fetched_requests) == 10
# Start at 10 because each page contains 10 items and we're processing the second page
for repo_number, request in enumerate(fetched_requests, 10):
assert request['repo'] == repo_template.format(repo_number)
def test_set_state_no_duplicate(app, client, db, worker_auth_env):
data = {
'repo': 'https://github.com/release-engineering/retrodep.git',
'ref': 'c50b93a32df1c9d700e3e80996845bc2e13be848',
'pkg_managers': ['gomod'],
}
# flask_login.current_user is used in Request.from_json, which requires a request context
with app.test_request_context(environ_base=worker_auth_env):
request = Request.from_json(data)
db.session.add(request)
db.session.commit()
state = 'complete'
state_reason = 'Completed successfully'
payload = {'state': state, 'state_reason': state_reason}
for i in range(3):
patch_rv = client.patch('/api/v1/requests/1',
json=payload,
environ_base=worker_auth_env)
assert patch_rv.status_code == 200
get_rv = client.get('/api/v1/requests/1')
assert get_rv.status_code == 200
fetched_request = json.loads(get_rv.data.decode('utf-8'))
# Make sure no duplicate states were added
assert len(fetched_request['state_history']) == 2
def test_set_state(app, client, db, worker_auth_env):
data = {
'repo': 'https://github.com/release-engineering/retrodep.git',
'ref': 'c50b93a32df1c9d700e3e80996845bc2e13be848',
'pkg_managers': ['gomod'],
}
# flask_login.current_user is used in Request.from_json, which requires a request context
with app.test_request_context(environ_base=worker_auth_env):
request = Request.from_json(data)
db.session.add(request)
db.session.commit()
state = 'complete'
state_reason = 'Completed successfully'
payload = {'state': state, 'state_reason': state_reason}
patch_rv = client.patch('/api/v1/requests/1',
json=payload,
environ_base=worker_auth_env)
assert patch_rv.status_code == 200
get_rv = client.get('/api/v1/requests/1')
assert get_rv.status_code == 200
fetched_request = json.loads(get_rv.data.decode('utf-8'))
assert fetched_request['state'] == state
assert fetched_request['state_reason'] == state_reason
# Since the date is always changing, the actual value can't be confirmed
assert fetched_request['updated']
assert len(fetched_request['state_history']) == 2
# Make sure the order is from newest to oldest
assert fetched_request['state_history'][0]['state'] == state
assert fetched_request['state_history'][0]['state_reason'] == state_reason
assert fetched_request['state_history'][0]['updated']
assert fetched_request['state_history'][1]['state'] == 'in_progress'
def test_set_deps(app, client, db, worker_auth_env, sample_deps, env_vars):
data = {
'repo': 'https://github.com/release-engineering/retrodep.git',
'ref': 'c50b93a32df1c9d700e3e80996845bc2e13be848',
'pkg_managers': ['gomod'],
}
# flask_login.current_user is used in Request.from_json, which requires a request context
with app.test_request_context(environ_base=worker_auth_env):
request = Request.from_json(data)
db.session.add(request)
db.session.commit()
payload = {'dependencies': sample_deps, 'environment_variables': env_vars}
patch_rv = client.patch('/api/v1/requests/1',
json=payload,
environ_base=worker_auth_env)
assert patch_rv.status_code == 200
len(EnvironmentVariable.query.all()) == len(env_vars.items())
for name, value in env_vars.items():
env_var_obj = EnvironmentVariable.query.filter_by(name=name,
value=value).first()
assert env_var_obj
get_rv = client.get('/api/v1/requests/1')
assert get_rv.status_code == 200
fetched_request = json.loads(get_rv.data.decode('utf-8'))
assert fetched_request['dependencies'] == sample_deps
assert fetched_request['environment_variables'] == env_vars
def test_set_state_stale(mock_remove, mock_exists, bundle_exists, app, client,
db, worker_auth_env):
mock_exists.return_value = bundle_exists
data = {
'repo': 'https://github.com/release-engineering/retrodep.git',
'ref': 'c50b93a32df1c9d700e3e80996845bc2e13be848',
'pkg_managers': ['gomod'],
}
# flask_login.current_user is used in Request.from_json, which requires a request context
with app.test_request_context(environ_base=worker_auth_env):
request = Request.from_json(data)
db.session.add(request)
db.session.commit()
state = 'stale'
state_reason = 'The request has expired'
payload = {'state': state, 'state_reason': state_reason}
patch_rv = client.patch('/api/v1/requests/1',
json=payload,
environ_base=worker_auth_env)
assert patch_rv.status_code == 200
get_rv = client.get('/api/v1/requests/1')
assert get_rv.status_code == 200
fetched_request = get_rv.get_json()
assert fetched_request['state'] == state
assert fetched_request['state_reason'] == state_reason
if bundle_exists:
mock_remove.assert_called_once_with(
'/tmp/cachito-archives/bundles/1.tar.gz')
else:
mock_remove.assert_not_called()
def create_request():
"""
Submit a request to resolve and cache the given source code and its dependencies.
:param str repo: the URL to the SCM repository
:param str ref: the SCM reference to fetch
:param list<str> pkg_managers: list of package managers to be used for resolving dependencies
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError('The input data must be a JSON object')
request = Request.from_json(payload)
if not re.match(r'^[a-f0-9]{40}', request.ref):
raise ValidationError(
'The "ref" parameter must be a 40 character hex string')
db.session.add(request)
db.session.commit()
if current_user.is_authenticated:
flask.current_app.logger.info('The user %s submitted request %d',
current_user.username, request.id)
else:
flask.current_app.logger.info('An anonymous user submitted request %d',
request.id)
db.session.add(request)
db.session.commit()
# Chain tasks
error_callback = tasks.failed_request_callback.s(request.id)
chain(
tasks.fetch_app_source.s(
request.repo, request.ref,
request_id_to_update=request.id).on_error(error_callback),
tasks.fetch_gomod_source.s(
request_id_to_update=request.id).on_error(error_callback),
tasks.create_bundle_archive.s(
request_id=request.id).on_error(error_callback),
tasks.set_request_state.si(request.id, 'complete',
'Completed successfully'),
).delay()
flask.current_app.logger.debug('Successfully scheduled request %d',
request.id)
return flask.jsonify(request.to_json()), 201
def test_state_change_invalid(app, client, db, worker_auth_env, request_id,
payload, status_code, message):
data = {
'repo': 'https://github.com/release-engineering/retrodep.git',
'ref': 'c50b93a32df1c9d700e3e80996845bc2e13be848',
'pkg_managers': ['gomod'],
}
# flask_login.current_user is used in Request.from_json, which requires a request context
with app.test_request_context(environ_base=worker_auth_env):
request = Request.from_json(data)
db.session.add(request)
db.session.commit()
rv = client.patch(f'/api/v1/requests/{request_id}',
json=payload,
environ_base=worker_auth_env)
assert rv.status_code == status_code
assert rv.json == {'error': message}
def test_add_dep_twice_diff_replaces(app, client, db, worker_auth_env):
data = {
'repo': 'https://github.com/release-engineering/retrodep.git',
'ref': 'c50b93a32df1c9d700e3e80996845bc2e13be848',
'pkg_managers': ['gomod'],
}
# flask_login.current_user is used in Request.from_json, which requires a request context
with app.test_request_context(environ_base=worker_auth_env):
request = Request.from_json(data)
db.session.add(request)
db.session.commit()
payload = {
'dependencies': [{
'name': 'all_systems_go',
'type': 'gomod',
'version': 'v1.0.0',
}]
}
patch_rv = client.patch('/api/v1/requests/1',
json=payload,
environ_base=worker_auth_env)
assert patch_rv.status_code == 200
# Add the dependency again with replaces set this time
payload2 = {
'dependencies': [{
'name': 'all_systems_go',
'type': 'gomod',
'replaces': {
'name': 'all_systems_go',
'type': 'gomod',
'version': 'v1.1.0',
},
'version': 'v1.0.0',
}]
}
patch_rv = client.patch('/api/v1/requests/1',
json=payload2,
environ_base=worker_auth_env)
assert patch_rv.status_code == 400
assert 'can\'t have a new replacement set' in patch_rv.json['error']
def test_set_deps(app, client, db, worker_auth_env, sample_deps_replace,
env_vars):
data = {
'repo': 'https://github.com/release-engineering/retrodep.git',
'ref': 'c50b93a32df1c9d700e3e80996845bc2e13be848',
'pkg_managers': ['gomod'],
}
# flask_login.current_user is used in Request.from_json, which requires a request context
with app.test_request_context(environ_base=worker_auth_env):
request = Request.from_json(data)
db.session.add(request)
db.session.commit()
# Test a dependency with no "replaces" key
sample_deps_replace.append({
'name': 'all_systems_go',
'type': 'gomod',
'version': 'v1.0.0',
})
payload = {
'dependencies': sample_deps_replace,
'environment_variables': env_vars
}
patch_rv = client.patch('/api/v1/requests/1',
json=payload,
environ_base=worker_auth_env)
assert patch_rv.status_code == 200
len(EnvironmentVariable.query.all()) == len(env_vars.items())
for name, value in env_vars.items():
env_var_obj = EnvironmentVariable.query.filter_by(name=name,
value=value).first()
assert env_var_obj
get_rv = client.get('/api/v1/requests/1')
assert get_rv.status_code == 200
fetched_request = get_rv.json
# Add a null "replaces" key to match the API output
sample_deps_replace[-1]['replaces'] = None
assert fetched_request['dependencies'] == sample_deps_replace
assert fetched_request['environment_variables'] == env_vars
def test_set_packages(app, client, db, sample_package, worker_auth_env):
data = {
'repo': 'https://github.com/release-engineering/retrodep.git',
'ref': 'c50b93a32df1c9d700e3e80996845bc2e13be848',
}
# flask_login.current_user is used in Request.from_json, which requires a request context
with app.test_request_context(environ_base=worker_auth_env):
request = Request.from_json(data)
db.session.add(request)
db.session.commit()
payload = {'packages': [sample_package]}
patch_rv = client.patch('/api/v1/requests/1',
json=payload,
environ_base=worker_auth_env)
assert patch_rv.status_code == 200
get_rv = client.get('/api/v1/requests/1')
assert get_rv.status_code == 200
assert get_rv.json['packages'] == [sample_package]
def test_set_deps(app, client, db, worker_auth_env, sample_deps):
data = {
'repo': 'https://github.com/release-engineering/retrodep.git',
'ref': 'c50b93a32df1c9d700e3e80996845bc2e13be848',
'pkg_managers': ['gomod'],
}
# flask_login.current_user is used in Request.from_json, which requires a request context
with app.test_request_context(environ_base=worker_auth_env):
request = Request.from_json(data)
db.session.add(request)
db.session.commit()
payload = {'dependencies': sample_deps}
patch_rv = client.patch('/api/v1/requests/1',
json=payload,
environ_base=worker_auth_env)
assert patch_rv.status_code == 200
get_rv = client.get('/api/v1/requests/1')
assert get_rv.status_code == 200
fetched_request = json.loads(get_rv.data.decode('utf-8'))
assert fetched_request['dependencies'] == sample_deps
def test_set_state_from_stale(app, client, db, worker_auth_env):
data = {
'repo': 'https://github.com/release-engineering/retrodep.git',
'ref': 'c50b93a32df1c9d700e3e80996845bc2e13be848',
'pkg_managers': ['gomod'],
}
# flask_login.current_user is used in Request.from_json, which requires a request context
with app.test_request_context(environ_base=worker_auth_env):
request = Request.from_json(data)
db.session.add(request)
db.session.commit()
request.add_state('stale', 'The request has expired')
db.session.commit()
payload = {'state': 'complete', 'state_reason': 'Unexpired'}
patch_rv = client.patch('/api/v1/requests/1',
json=payload,
environ_base=worker_auth_env)
assert patch_rv.status_code == 400
assert patch_rv.get_json() == {
'error': 'A stale request cannot change states'
}
def test_set_state(mock_rmtree, mock_exists, state, app, client, db,
worker_auth_env):
mock_exists.return_value = True
data = {
'repo': 'https://github.com/release-engineering/retrodep.git',
'ref': 'c50b93a32df1c9d700e3e80996845bc2e13be848',
'pkg_managers': ['gomod'],
}
# flask_login.current_user is used in Request.from_json, which requires a request context
with app.test_request_context(environ_base=worker_auth_env):
request = Request.from_json(data)
db.session.add(request)
db.session.commit()
state = state
state_reason = 'Some status'
payload = {'state': state, 'state_reason': state_reason}
patch_rv = client.patch('/api/v1/requests/1',
json=payload,
environ_base=worker_auth_env)
assert patch_rv.status_code == 200
get_rv = client.get('/api/v1/requests/1')
assert get_rv.status_code == 200
fetched_request = get_rv.json
assert fetched_request['state'] == state
assert fetched_request['state_reason'] == state_reason
# Since the date is always changing, the actual value can't be confirmed
assert fetched_request['updated']
assert len(fetched_request['state_history']) == 2
# Make sure the order is from newest to oldest
assert fetched_request['state_history'][0]['state'] == state
assert fetched_request['state_history'][0]['state_reason'] == state_reason
assert fetched_request['state_history'][0]['updated']
assert fetched_request['state_history'][1]['state'] == 'in_progress'
mock_exists.assert_called_once_with('/tmp/cachito-archives/bundles/temp/1')
mock_rmtree.assert_called_once_with('/tmp/cachito-archives/bundles/temp/1')
def create_request():
"""
Submit a request to resolve and cache the given source code and its dependencies.
:param str repo: the URL to the SCM repository
:param str ref: the SCM reference to fetch
:param list<str> pkg_managers: list of package managers to be used for resolving dependencies
:param list<str> flags: list of flag names
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError('The input data must be a JSON object')
request = Request.from_json(payload)
if not re.match(r'^[a-f0-9]{40}', request.ref):
raise ValidationError(
'The "ref" parameter must be a 40 character hex string')
db.session.add(request)
db.session.commit()
if current_user.is_authenticated:
flask.current_app.logger.info('The user %s submitted request %d',
current_user.username, request.id)
else:
flask.current_app.logger.info('An anonymous user submitted request %d',
request.id)
pkg_manager_names = set(pkg_manager.name
for pkg_manager in request.pkg_managers)
auto_detect = len(pkg_manager_names) == 0
if auto_detect:
flask.current_app.logger.info(
'Automatic detection will be used since "pkg_managers" was empty')
# Chain tasks
error_callback = tasks.failed_request_callback.s(request.id)
chain_tasks = [
tasks.fetch_app_source.s(request.repo, request.ref,
request.id).on_error(error_callback),
]
if 'gomod' in pkg_manager_names or auto_detect:
gomod_dependency_replacements = [
dependency_replacement for dependency_replacement in payload.get(
'dependency_replacements', [])
if dependency_replacement['type'] == 'gomod'
]
chain_tasks.append(
tasks.fetch_gomod_source.si(
request.id,
auto_detect,
gomod_dependency_replacements,
).on_error(error_callback))
chain_tasks.extend([
tasks.create_bundle_archive.si(request.id).on_error(error_callback),
tasks.set_request_state.si(request.id, 'complete',
'Completed successfully'),
])
chain(chain_tasks).delay()
flask.current_app.logger.debug('Successfully scheduled request %d',
request.id)
return flask.jsonify(request.to_json()), 201
def create_request():
"""
Submit a request to resolve and cache the given source code and its dependencies.
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError("The input data must be a JSON object")
request = Request.from_json(payload)
db.session.add(request)
db.session.commit()
if current_user.is_authenticated:
flask.current_app.logger.info("The user %s submitted request %d",
current_user.username, request.id)
else:
flask.current_app.logger.info("An anonymous user submitted request %d",
request.id)
pkg_manager_names = set(pkg_manager.name
for pkg_manager in request.pkg_managers)
supported_pkg_managers = set(
flask.current_app.config["CACHITO_PACKAGE_MANAGERS"])
unsupported_pkg_managers = pkg_manager_names - supported_pkg_managers
if unsupported_pkg_managers:
# At this point, unsupported_pkg_managers would only contain valid package managers that
# are not enabled
raise ValidationError(
"The following package managers are not "
f"enabled: {', '.join(unsupported_pkg_managers)}")
# Chain tasks
error_callback = tasks.failed_request_callback.s(request.id)
chain_tasks = [
tasks.fetch_app_source.s(request.repo, request.ref, request.id,
"git-submodule"
in pkg_manager_names).on_error(error_callback)
]
pkg_manager_to_dep_replacements = {}
for dependency_replacement in payload.get("dependency_replacements", []):
type_ = dependency_replacement["type"]
pkg_manager_to_dep_replacements.setdefault(type_, [])
pkg_manager_to_dep_replacements[type_].append(dependency_replacement)
package_configs = payload.get("packages", {})
if "gomod" in pkg_manager_names:
go_package_configs = package_configs.get("gomod", [])
chain_tasks.append(
tasks.fetch_gomod_source.si(
request.id, pkg_manager_to_dep_replacements.get("gomod", []),
go_package_configs).on_error(error_callback))
if "npm" in pkg_manager_names:
if pkg_manager_to_dep_replacements.get("npm"):
raise ValidationError(
"Dependency replacements are not yet supported for the npm package manager"
)
npm_package_configs = package_configs.get("npm", [])
chain_tasks.append(
tasks.fetch_npm_source.si(
request.id, npm_package_configs).on_error(error_callback))
if "pip" in pkg_manager_names:
if pkg_manager_to_dep_replacements.get("pip"):
raise ValidationError(
"Dependency replacements are not yet supported for the pip package manager"
)
pip_package_configs = package_configs.get("pip", [])
chain_tasks.append(
tasks.fetch_pip_source.si(
request.id, pip_package_configs).on_error(error_callback))
if "git-submodule" in pkg_manager_names:
chain_tasks.append(
tasks.add_git_submodules_as_package.si(
request.id).on_error(error_callback))
if "yarn" in pkg_manager_names:
if pkg_manager_to_dep_replacements.get("yarn"):
raise ValidationError(
"Dependency replacements are not yet supported for the yarn package manager"
)
yarn_package_configs = package_configs.get("yarn", [])
chain_tasks.append(
tasks.fetch_yarn_source.si(
request.id, yarn_package_configs).on_error(error_callback))
chain_tasks.append(
tasks.create_bundle_archive.si(request.id).on_error(error_callback))
try:
chain(chain_tasks).delay()
except kombu.exceptions.OperationalError:
flask.current_app.logger.exception(
"Failed to schedule the task for request %d. Failing the request.",
request.id)
error = "Failed to schedule the task to the workers. Please try again."
request.add_state("failed", error)
raise CachitoError(error)
flask.current_app.logger.debug("Successfully scheduled request %d",
request.id)
return flask.jsonify(request.to_json()), 201
def test_fetch_paginated_requests(
mock_chain,
app,
auth_env,
client,
db,
sample_deps_replace,
sample_package,
worker_auth_env,
):
repo_template = 'https://github.com/release-engineering/retrodep{}.git'
# flask_login.current_user is used in Request.from_json, which requires a request context
with app.test_request_context(environ_base=auth_env):
for i in range(50):
data = {
'repo': repo_template.format(i),
'ref': 'c50b93a32df1c9d700e3e80996845bc2e13be848',
'pkg_managers': ['gomod'],
}
request = Request.from_json(data)
db.session.add(request)
db.session.commit()
payload = {
'dependencies': sample_deps_replace,
'packages': [sample_package]
}
client.patch('/api/v1/requests/1',
json=payload,
environ_base=worker_auth_env)
client.patch('/api/v1/requests/11',
json=payload,
environ_base=worker_auth_env)
# Sane defaults are provided
rv = client.get('/api/v1/requests')
assert rv.status_code == 200
response = rv.json
fetched_requests = response['items']
assert len(fetched_requests) == 20
for repo_number, request in enumerate(fetched_requests):
assert request['repo'] == repo_template.format(repo_number)
assert response['meta']['previous'] is None
assert fetched_requests[0]['dependencies'] == 14
assert fetched_requests[0]['packages'] == 1
# per_page and page parameters are honored
rv = client.get('/api/v1/requests?page=2&per_page=10&verbose=True')
assert rv.status_code == 200
response = rv.json
fetched_requests = response['items']
assert len(fetched_requests) == 10
# Start at 10 because each page contains 10 items and we're processing the second page
for repo_number, request in enumerate(fetched_requests, 10):
assert request['repo'] == repo_template.format(repo_number)
pagination_metadata = response['meta']
for page, page_num in [('next', 3), ('last', 5), ('previous', 1),
('first', 1)]:
assert f'page={page_num}' in pagination_metadata[page]
assert 'per_page=10' in pagination_metadata[page]
assert 'verbose=True' in pagination_metadata[page]
assert pagination_metadata['total'] == 50
assert len(fetched_requests[0]['dependencies']) == 14
assert len(fetched_requests[0]['packages']) == 1
assert type(fetched_requests[0]['dependencies']) == list