def queue_scan() -> Tuple[Dict, int]: """ Queues a scan for package version or versions and returns the scan JSON with status 202 """ body = request.get_json() log.warning(f"received scan JSON body: {body}") if (isinstance(body, dict) and "scan_type" in body and body["scan_type"] == "scan_score_npm_dep_files"): schema = ScanScoreNPMDepFilesRequestParamsSchema loader = schema_to_dep_files_scan else: schema = ScanScoreNPMPackageRequestParamsSchema loader = schema_to_package_scan try: scan_config = schema().load(data=body) except ValidationError as err: return err.messages, 422 log.info( f"deserialized scan JSON to {scan_config.scan_type}: {scan_config}" ) # type: ignore if scan_config.scan_type not in current_app.config[ "WEB_JOB_NAMES"]: # type: ignore raise BadRequest( description="scan type not allowed or does not exist for app") scan = models.save_scan_with_status( loader(scan_config), ScanStatusEnum["queued"], ) log.info(f"queued scan {scan.id}") return ScanSchema().dump(scan), 202
def cancel_scan(scan_id: int) -> Dict: """ Cancels the scan and returns it as JSON Does not cancel scan jobs. """ scan = models.get_scan_by_id(scan_id).one() if scan.status != ScanStatusEnum["queued"]: raise BadRequest(description="Cannot cancel un-queued scan {scan_id}.") models.save_scan_with_status( scan, ScanStatusEnum["canceled"], ) log.info(f"canceled scan {scan_id}") return ScanSchema().dump(models.get_scan_by_id(scan_id).one())
async def run_scan( app: flask.Flask, scan: models.Scan, ) -> models.Scan: """ Async task that: * takes a scan job * starts a k8s job in the untrusted jobs cluster * updates the scan status from 'queued' to 'started' * watches the k8s job and sets the scan status to 'failed' or 'succeeded' when the k8s job finishes Returns the updated scan. """ if not (isinstance(scan.params, dict) and all(k in scan.params.keys() for k in {"name", "args", "kwargs"})): log.info(f"ignoring pending scan {scan.id} with params {scan.params}") return scan assert scan.name in { "scan_score_npm_dep_files", "scan_score_npm_package", } scan_fn = getattr(scans, scan.name) if scan.name != scan_fn.__name__: raise NotImplementedError(f"Scan of type {scan.name} not implemented") log.info( f"starting a k8s job for {scan.name} scan {scan.id} with params {scan.params} using {scan_fn}" ) with app.app_context(): scan = models.save_scan_with_status(scan, ScanStatusEnum["started"]) # scan fails if any of its tarball scan jobs, data fetching, or scoring steps fail try: await scan_fn(scan) new_scan_status = ScanStatusEnum["succeeded"] except Exception as err: log.error( f"{scan.id} error scanning and scoring: {err}\n{exc_to_str()}") new_scan_status = ScanStatusEnum["failed"] return models.save_scan_with_status(scan, new_scan_status)
def scan_npm_package(package_name: str, package_version: str) -> None: """ Scan and score an npm package name and version """ scan = models.save_scan_with_status( models.package_name_and_version_to_scan(package_name, package_version), models.ScanStatusEnum["queued"], ) log.info(f"running npm package scan with id {scan.id}") asyncio.run(start_scan(scan)) log.info(f"started npm package scan") while True: asyncio.run(finish_scan(scan)) log.info("waiting for scan to finish") time.sleep(3)
async def start_scan(scan: models.Scan, ) -> models.Scan: """ Async task that: * takes a scan job * starts one or more k8s jobs in the untrusted jobs cluster * updates the scan status from 'queued' to 'started' and adds k8s job_names and returns the updated scan. Run in a flask app context. """ try: if not (isinstance(scan.params, dict) and all(k in scan.params.keys() for k in {"name", "args", "kwargs"})): raise Exception( f"queued scan {scan.id} has invalid params {scan.params}") scan_config = scan_type_to_config(scan.name) log.info( f"starting k8s jobs for {scan.name} scan {scan.id} with params {scan.params}" ) job_configs: Dict[str, k8s.KubeJobConfig] = {} async for job_config in scan_config.job_configs(scan): job_configs[job_config["name"]] = job_config models.save_scan_with_job_names(scan, list(job_configs.keys())) for job_name, job_config in job_configs.items(): log.info( f"scan {scan.id} starting k8s job {job_name} with config {job_config}" ) k8s.create_job(job_config) log.info(f"scan {scan.id} started k8s job {job_name}") new_scan_status = ScanStatusEnum["started"] except Exception as err: log.error(f"{scan.id} error starting k8s jobs: {err}\n{exc_to_str()}") new_scan_status = ScanStatusEnum["failed"] started_scan = models.save_scan_with_status(scan, new_scan_status) assert started_scan.status in { ScanStatusEnum["started"], ScanStatusEnum["failed"], } log.info(f"scan {scan.id} updated status to {started_scan.status}") return started_scan