def queue_scan() -> Tuple[Dict, int]:
"""
Queues a scan for package version or versions and returns the scan
JSON with status 202
"""
body = request.get_json()
log.warning(f"received scan JSON body: {body}")
if (isinstance(body, dict) and "scan_type" in body
and body["scan_type"] == "scan_score_npm_dep_files"):
schema = ScanScoreNPMDepFilesRequestParamsSchema
loader = schema_to_dep_files_scan
else:
schema = ScanScoreNPMPackageRequestParamsSchema
loader = schema_to_package_scan
try:
scan_config = schema().load(data=body)
except ValidationError as err:
return err.messages, 422
log.info(
f"deserialized scan JSON to {scan_config.scan_type}: {scan_config}"
) # type: ignore
if scan_config.scan_type not in current_app.config[
"WEB_JOB_NAMES"]: # type: ignore
raise BadRequest(
description="scan type not allowed or does not exist for app")
scan = models.save_scan_with_status(
loader(scan_config),
ScanStatusEnum["queued"],
)
log.info(f"queued scan {scan.id}")
return ScanSchema().dump(scan), 202
def cancel_scan(scan_id: int) -> Dict:
"""
Cancels the scan and returns it as JSON
Does not cancel scan jobs.
"""
scan = models.get_scan_by_id(scan_id).one()
if scan.status != ScanStatusEnum["queued"]:
raise BadRequest(description="Cannot cancel un-queued scan {scan_id}.")
models.save_scan_with_status(
scan,
ScanStatusEnum["canceled"],
)
log.info(f"canceled scan {scan_id}")
return ScanSchema().dump(models.get_scan_by_id(scan_id).one())
async def run_scan(
app: flask.Flask,
scan: models.Scan,
) -> models.Scan:
"""
Async task that:
* takes a scan job
* starts a k8s job in the untrusted jobs cluster
* updates the scan status from 'queued' to 'started'
* watches the k8s job and sets the scan status to 'failed' or 'succeeded' when the k8s job finishes
Returns the updated scan.
"""
if not (isinstance(scan.params, dict)
and all(k in scan.params.keys()
for k in {"name", "args", "kwargs"})):
log.info(f"ignoring pending scan {scan.id} with params {scan.params}")
return scan
assert scan.name in {
"scan_score_npm_dep_files",
"scan_score_npm_package",
}
scan_fn = getattr(scans, scan.name)
if scan.name != scan_fn.__name__:
raise NotImplementedError(f"Scan of type {scan.name} not implemented")
log.info(
f"starting a k8s job for {scan.name} scan {scan.id} with params {scan.params} using {scan_fn}"
)
with app.app_context():
scan = models.save_scan_with_status(scan, ScanStatusEnum["started"])
# scan fails if any of its tarball scan jobs, data fetching, or scoring steps fail
try:
await scan_fn(scan)
new_scan_status = ScanStatusEnum["succeeded"]
except Exception as err:
log.error(
f"{scan.id} error scanning and scoring: {err}\n{exc_to_str()}")
new_scan_status = ScanStatusEnum["failed"]
return models.save_scan_with_status(scan, new_scan_status)
def scan_npm_package(package_name: str, package_version: str) -> None:
"""
Scan and score an npm package name and version
"""
scan = models.save_scan_with_status(
models.package_name_and_version_to_scan(package_name, package_version),
models.ScanStatusEnum["queued"],
)
log.info(f"running npm package scan with id {scan.id}")
asyncio.run(start_scan(scan))
log.info(f"started npm package scan")
while True:
asyncio.run(finish_scan(scan))
log.info("waiting for scan to finish")
time.sleep(3)
async def start_scan(scan: models.Scan, ) -> models.Scan:
"""
Async task that:
* takes a scan job
* starts one or more k8s jobs in the untrusted jobs cluster
* updates the scan status from 'queued' to 'started' and adds k8s job_names
and returns the updated scan.
Run in a flask app context.
"""
try:
if not (isinstance(scan.params, dict)
and all(k in scan.params.keys()
for k in {"name", "args", "kwargs"})):
raise Exception(
f"queued scan {scan.id} has invalid params {scan.params}")
scan_config = scan_type_to_config(scan.name)
log.info(
f"starting k8s jobs for {scan.name} scan {scan.id} with params {scan.params}"
)
job_configs: Dict[str, k8s.KubeJobConfig] = {}
async for job_config in scan_config.job_configs(scan):
job_configs[job_config["name"]] = job_config
models.save_scan_with_job_names(scan, list(job_configs.keys()))
for job_name, job_config in job_configs.items():
log.info(
f"scan {scan.id} starting k8s job {job_name} with config {job_config}"
)
k8s.create_job(job_config)
log.info(f"scan {scan.id} started k8s job {job_name}")
new_scan_status = ScanStatusEnum["started"]
except Exception as err:
log.error(f"{scan.id} error starting k8s jobs: {err}\n{exc_to_str()}")
new_scan_status = ScanStatusEnum["failed"]
started_scan = models.save_scan_with_status(scan, new_scan_status)
assert started_scan.status in {
ScanStatusEnum["started"],
ScanStatusEnum["failed"],
}
log.info(f"scan {scan.id} updated status to {started_scan.status}")
return started_scan