def validate_application_files(application_path: pathlib.Path):
"""
Validate application files at a given directory.
Confirms:
application_path exists
application_path contains an application python module
application_path contains an application configuration file
"""
with Abort.check_expressions(
f"The application files in {application_path} were invalid",
raise_kwargs=dict(
subject="Invalid application files",
log_message=
f"Application files located at {application_path} failed validation",
),
) as checker:
checker(
application_path.exists(),
f"Application directory {application_path} does not exist",
)
application_module = application_path / JOBBERGATE_APPLICATION_MODULE_FILE_NAME
checker(
application_module.exists(),
unwrap(f"""
Application directory does not contain required application module
{JOBBERGATE_APPLICATION_MODULE_FILE_NAME}
"""),
)
try:
ast.parse(application_module.read_text())
is_valid_python = True
except Exception:
is_valid_python = False
checker(
is_valid_python,
f"The application module at {application_module} is not valid python code"
)
application_config = application_path / JOBBERGATE_APPLICATION_CONFIG_FILE_NAME
checker(
application_config.exists(),
unwrap(f"""
Application directory does not contain required configuration file
{JOBBERGATE_APPLICATION_MODULE_FILE_NAME}
"""),
)
try:
yaml.safe_load(application_config.read_text())
is_valid_yaml = True
except Exception:
is_valid_yaml = False
checker(
is_valid_yaml,
f"The application config at {application_config} is not valid YAML"
)
def test_update__warns_but_does_not_abort_if_upload_fails(
respx_mock,
make_test_app,
dummy_context,
dummy_application_data,
dummy_application_dir,
dummy_domain,
cli_runner,
mocker,
):
response_data = dummy_application_data[0]
response_data["application_uploaded"] = False
application_id = response_data["id"]
update_route = respx_mock.put(
f"{dummy_domain}/applications/{application_id}")
update_route.mock(return_value=httpx.Response(
httpx.codes.OK,
json=response_data,
), )
upload_route = respx_mock.post(
f"{dummy_domain}/applications/{application_id}/upload")
upload_route.mock(return_value=httpx.Response(httpx.codes.BAD_REQUEST), )
test_app = make_test_app("update", update)
mocked_render = mocker.patch(
"jobbergate_cli.subapps.applications.app.render_single_result")
result = cli_runner.invoke(
test_app,
shlex.split(
unwrap(f"""
update --id={application_id} --identifier=dummy-identifier
--application-path={dummy_application_dir}
--application-desc="This application is kinda dumb, actually"
""")),
)
assert result.exit_code == 0, f"update failed: {result.stdout}"
assert update_route.called
assert upload_route.called
assert "zipped application files could not be uploaded" in result.stdout
mocked_render.assert_called_once_with(
dummy_context,
{
**response_data, "application_uploaded": False
},
title="Updated Application",
hidden_fields=HIDDEN_FIELDS,
)
def test_create__success(
respx_mock,
make_test_app,
dummy_context,
dummy_application_data,
dummy_application_dir,
dummy_domain,
cli_runner,
mocker,
):
response_data = dummy_application_data[0]
response_data["application_uploaded"] = False
application_id = response_data["id"]
create_route = respx_mock.post(f"{dummy_domain}/applications")
create_route.mock(return_value=httpx.Response(
httpx.codes.CREATED,
json=response_data,
), )
upload_route = respx_mock.post(
f"{dummy_domain}/applications/{application_id}/upload")
upload_route.mock(return_value=httpx.Response(httpx.codes.CREATED))
test_app = make_test_app("create", create)
mocked_render = mocker.patch(
"jobbergate_cli.subapps.applications.app.render_single_result")
result = cli_runner.invoke(
test_app,
shlex.split(
unwrap(f"""
create --name=dummy-name --identifier=dummy-identifier
--application-path={dummy_application_dir}
--application-desc="This application is kinda dumb, actually"
""")),
)
assert result.exit_code == 0, f"create failed: {result.stdout}"
assert create_route.called
assert upload_route.called
mocked_render.assert_called_once_with(
dummy_context,
{
**response_data, "application_uploaded": True
},
title="Created Application",
hidden_fields=HIDDEN_FIELDS,
)
def test_update__makes_request_and_renders_results(
respx_mock,
make_test_app,
dummy_context,
dummy_job_script_data,
dummy_domain,
cli_runner,
mocker,
):
job_script_data = dummy_job_script_data[0]
job_script_id = job_script_data["id"]
new_job_script_data_as_string = json.dumps({"application.sh": "#!bin/bash \\n echo so dumb"})
new_job_script_data = {
**job_script_data,
"job_script_data_as_string": new_job_script_data_as_string,
}
respx_mock.put(f"{dummy_domain}/job-scripts/{job_script_id}").mock(
return_value=httpx.Response(httpx.codes.OK, json=new_job_script_data),
)
test_app = make_test_app("update", update)
mocked_render = mocker.patch("jobbergate_cli.subapps.job_scripts.app.render_single_result")
result = cli_runner.invoke(
test_app,
shlex.split(
unwrap(
f"""
update --id={job_script_id}
--job-script='{new_job_script_data_as_string}'
"""
)
),
)
assert result.exit_code == 0, f"update failed: {result.stdout}"
mocked_render.assert_called_once_with(
dummy_context,
JobScriptResponse(**new_job_script_data),
title="Updated Job Script",
hidden_fields=HIDDEN_FIELDS,
)
def wrapper(*args, **kwargs):
try:
func(*args, **kwargs)
except Abort as err:
if not err.warn_only:
if err.log_message is not None:
logger.error(err.log_message)
if err.original_error is not None:
logger.error(f"Original exception: {err.original_error}")
if settings.SENTRY_DSN:
with sentry_sdk.push_scope() as scope:
if err.sentry_context is not None:
scope.set_context(**err.sentry_context)
sentry_sdk.capture_exception(err.original_error if err.original_error is not None else err)
sentry_sdk.flush()
panel_kwargs = dict()
if err.subject is not None:
panel_kwargs["title"] = f"[red]{err.subject}"
message = dedent(err.message)
if err.support:
support_message = unwrap(
f"""
[yellow]If the problem persists,
please contact [bold]{OV_CONTACT}[/bold]
for support and trouble-shooting
"""
)
message = f"{message}\n\n{support_message}"
console = Console()
console.print()
console.print(Panel(message, **panel_kwargs))
console.print()
raise typer.Exit(code=1)
def test_create(
make_test_app,
dummy_context,
dummy_job_submission_data,
cli_runner,
mocker,
):
job_submission_data = JobSubmissionResponse(**dummy_job_submission_data[0])
job_submission_name = job_submission_data.job_submission_name
job_submission_description = job_submission_data.job_submission_description
job_script_id = job_submission_data.id
mocked_render = mocker.patch(
"jobbergate_cli.subapps.job_submissions.app.render_single_result")
patched_create_job_submission = mocker.patch(
"jobbergate_cli.subapps.job_submissions.app.create_job_submission")
patched_create_job_submission.return_value = job_submission_data
test_app = make_test_app("create", create)
result = cli_runner.invoke(
test_app,
shlex.split(
unwrap(f"""
create --name={job_submission_name}
--description='{job_submission_description}'
--job-script-id={job_script_id}
""")),
)
assert result.exit_code == 0, f"create failed: {result.stdout}"
mocked_render.assert_called_once_with(
dummy_context,
job_submission_data,
title="Created Job Submission",
hidden_fields=HIDDEN_FIELDS,
)
Provide a stub module to maintain compatibility with previous versions.
Issue a deprecation warning when this module is imported from if JOBBERGATE_COMPATIBILITY_MODE is enabled.
If JOBBERGATE_COMPATIBILITY_MODE is not enabled, raise an import error when this module is imported.
"""
import warnings
from jobbergate_cli.config import settings
from jobbergate_cli.text_tools import dedent, unwrap
if settings.JOBBERGATE_COMPATIBILITY_MODE:
from jobbergate_cli.subapps.applications.application_base import JobbergateApplicationBase # noqa
warnings.warn(
dedent("""
Importing application_base from jobbergate_cli is deprecated.
The module has been moved.
Import 'application_base' from 'jobbergate_cli.subapps.applications' instead",
"""),
DeprecationWarning,
)
else:
raise ImportError(
unwrap("""
JobbergateApplicationBase has been moved to
'jobbergate_cli.subapps.applications.application_base.JobbergateApplicationBase'
"""))
def fetch_auth_tokens(ctx: JobbergateContext) -> TokenSet:
"""
Fetch an access token (and possibly a refresh token) from Auth0.
Prints out a URL for the user to use to authenticate and polls the token endpoint to fetch it when
the browser-based process finishes
"""
# Make static type-checkers happy
assert ctx.client is not None
device_code_data = cast(
DeviceCodeData,
make_request(
ctx.client,
"/oauth/device/code",
"POST",
expected_status=200,
abort_message="There was a problem retrieving a device verification code from the auth provider",
abort_subject="COULD NOT RETRIEVE TOKEN",
support=True,
response_model_cls=DeviceCodeData,
data=dict(
client_id=settings.AUTH0_CLIENT_ID,
audience=settings.AUTH0_AUDIENCE,
scope="offline_access", # To get refresh token
),
),
)
terminal_message(
f"""
To complete login, please open the following link in a browser:
{device_code_data.verification_uri_complete}
Waiting up to {settings.AUTH0_MAX_POLL_TIME / 60} minutes for you to complete the process...
""",
subject="Waiting for login",
)
for tick in TimeLoop(
settings.AUTH0_MAX_POLL_TIME,
message="Waiting for web login",
):
response_data = cast(
Dict,
make_request(
ctx.client,
"/oauth/token",
"POST",
abort_message="There was a problem retrieving a device verification code from the auth provider",
abort_subject="COULD NOT FETCH ACCESS TOKEN",
support=True,
data=dict(
grant_type="urn:ietf:params:oauth:grant-type:device_code",
device_code=device_code_data.device_code,
client_id=settings.AUTH0_CLIENT_ID,
),
),
)
if "error" in response_data:
if response_data["error"] == "authorization_pending":
logger.debug(f"Token fetch attempt #{tick.counter} failed")
sleep(device_code_data.interval)
else:
# TODO: Test this failure condition
raise Abort(
unwrap(
"""
There was a problem retrieving a device verification code from the auth provider:
Unexpected failure retrieving access token.
"""
),
subject="Unexpected error",
support=True,
log_message=f"Unexpected error response: {response_data}",
)
else:
return TokenSet(**response_data)
raise Abort(
"Login process was not completed in time. Please try again.",
subject="Timed out",
log_message="Timed out while waiting for user to complete login",
)
def test_create__non_fast_mode_and_job_submission(
respx_mock,
make_test_app,
dummy_context,
dummy_application_data,
dummy_job_script_data,
dummy_job_submission_data,
dummy_domain,
dummy_render_class,
cli_runner,
tmp_path,
attach_persona,
mocker,
):
application_response = ApplicationResponse(**dummy_application_data[0])
job_script_data = dummy_job_script_data[0]
job_script_id = job_script_data["id"]
job_submission_data = dummy_job_submission_data[0]
create_route = respx_mock.post(f"{dummy_domain}/job-scripts")
create_route.mock(
return_value=httpx.Response(
httpx.codes.CREATED,
json=job_script_data,
),
)
sbatch_params = " ".join(f"--sbatch-params={i}" for i in (1, 2, 3))
param_file_path = tmp_path / "param_file.json"
param_file_path.write_text(json.dumps(dict(foo="oof")))
dummy_render_class.prepared_input = dict(
foo="FOO",
bar="BAR",
baz="BAZ",
)
attach_persona("*****@*****.**")
test_app = make_test_app("create", create)
mocked_render = mocker.patch("jobbergate_cli.subapps.job_scripts.app.render_single_result")
mocked_fetch_application_data = mocker.patch(
"jobbergate_cli.subapps.job_scripts.app.fetch_application_data",
return_value=application_response,
)
mocked_create_job_submission = mocker.patch(
"jobbergate_cli.subapps.job_scripts.app.create_job_submission",
return_value=JobSubmissionResponse(**job_submission_data),
)
mocker.patch.object(
importlib.import_module("inquirer.prompt"),
"ConsoleRender",
new=dummy_render_class,
)
result = cli_runner.invoke(
test_app,
shlex.split(
unwrap(
f"""
create --name=dummy-name
--application-id={application_response.id}
--param-file={param_file_path}
{sbatch_params}
"""
)
),
input="y\n", # To confirm that the job should be submitted right away
)
assert result.exit_code == 0, f"create failed: {result.stdout}"
assert mocked_fetch_application_data.called_once_with(
dummy_context,
id=application_response.id,
identifier=None,
)
assert mocked_create_job_submission.called_once_with(
dummy_context,
job_script_id,
"dummy-name",
)
assert create_route.called
content = json.loads(create_route.calls.last.request.content)
assert content == dict(
param_dict=dict(
application_config=dict(
foo="oof",
bar="BAR",
baz="BAZ",
),
jobbergate_config=dict(
default_template="test-job-script.py.j2",
job_script_name=None,
output_directory=".",
supporting_files=None,
supporting_files_output_name=None,
template_files=["test-job-script.py.j2"],
),
),
application_id=application_response.id,
job_script_name="dummy-name",
sbatch_params=["1", "2", "3"],
)
mocked_render.assert_has_calls(
[
mocker.call(
dummy_context,
JobScriptResponse(**job_script_data),
title="Created Job Script",
hidden_fields=HIDDEN_FIELDS,
),
mocker.call(
dummy_context,
JobSubmissionResponse(**job_submission_data),
title="Created Job Submission (Fast Mode)",
hidden_fields=JOB_SUBMISSION_HIDDEN_FIELDS,
),
]
)
def test_create__with_fast_mode_and_no_job_submission(
respx_mock,
make_test_app,
dummy_context,
dummy_application_data,
dummy_job_script_data,
dummy_domain,
cli_runner,
tmp_path,
attach_persona,
mocker,
):
application_response = ApplicationResponse(**dummy_application_data[0])
job_script_data = dummy_job_script_data[0]
create_route = respx_mock.post(f"{dummy_domain}/job-scripts")
create_route.mock(
return_value=httpx.Response(
httpx.codes.CREATED,
json=job_script_data,
),
)
sbatch_params = " ".join(f"--sbatch-params={i}" for i in (1, 2, 3))
param_file_path = tmp_path / "param_file.json"
param_file_path.write_text(
json.dumps(
dict(
foo="oof",
bar="rab",
baz="zab",
)
)
)
attach_persona("*****@*****.**")
test_app = make_test_app("create", create)
mocked_render = mocker.patch("jobbergate_cli.subapps.job_scripts.app.render_single_result")
mocked_fetch_application_data = mocker.patch(
"jobbergate_cli.subapps.job_scripts.app.fetch_application_data",
return_value=application_response,
)
result = cli_runner.invoke(
test_app,
shlex.split(
unwrap(
f"""
create --name=dummy-name
--application-id={application_response.id}
--param-file={param_file_path}
--fast
--no-submit
{sbatch_params}
"""
)
),
)
assert result.exit_code == 0, f"create failed: {result.stdout}"
assert mocked_fetch_application_data.called_once_with(
dummy_context,
id=application_response.id,
identifier=None,
)
assert create_route.called
content = json.loads(create_route.calls.last.request.content)
assert content == dict(
param_dict=dict(
application_config=dict(
foo="oof",
bar="rab",
baz="zab",
),
jobbergate_config=dict(
default_template="test-job-script.py.j2",
job_script_name=None,
output_directory=".",
supporting_files=None,
supporting_files_output_name=None,
template_files=["test-job-script.py.j2"],
),
),
application_id=application_response.id,
job_script_name="dummy-name",
sbatch_params=["1", "2", "3"],
)
mocked_render.assert_called_once_with(
dummy_context,
JobScriptResponse(**job_script_data),
title="Created Job Script",
hidden_fields=HIDDEN_FIELDS,
)
Provide a stub module to maintain compatibility with previous versions.
Issue a deprecation warning when this module is imported from if JOBBERGATE_COMPATIBILITY_MODE is enabled.
If JOBBERGATE_COMPATIBILITY_MODE is not enabled, raise an import error when this module is imported.
"""
import warnings
from jobbergate_cli.config import settings
from jobbergate_cli.text_tools import dedent, unwrap
if settings.JOBBERGATE_COMPATIBILITY_MODE:
from jobbergate_cli.subapps.applications.application_helpers import * # noqa
warnings.warn(
dedent("""
Importing jobberappslib from jobbergate_cli is deprecated.
The module has been moved.
Import the helper functions from 'jobbergate_cli.subapps.applications.application_helpers' instead",
"""),
DeprecationWarning,
)
else:
raise ImportError(
unwrap("""
The 'jobberappslib' module has been renamed to 'application_helpers' and has been moved to
'jobbergate_cli.subapps.applications.application_helpers'
"""))