def save_pubsub_message(app: flask.Flask, message: gcp.pubsub_v1.types.PubsubMessage) -> None: """ Saves a pubsub message data to the JSONResult table and acks it. nacks it if saving fails. Requires depobs flask app context. """ with app.app_context(): try: # TODO: set job status when it finishes? No, do this in the runner. log.info( f"received pubsub message {message.message_id} published at {message.publish_time} with attrs {message.attributes}" ) save_json_results([{ "type": "google.cloud.pubsub_v1.types.PubsubMessage", "id": message.message_id, "publish_time": flask.json.dumps(message.publish_time), # convert datetime "attributes": dict(message.attributes), # convert from ScalarMapContainer "data": flask.json.loads(message.data), "size": message.size, }]) message.ack() except Exception as err: message.nack() log.error( f"error saving pubsub message {message} to json results table: {err}" )
def fetch_and_save_registry_entries( package_names: Iterable[str]) -> List[Dict]: package_names = list(package_names) log.info( f"fetching registry entries for {len(package_names)} package names") log.debug( f"fetching registry entries for package names: {list(package_names)}") npm_registry_entries = asyncio.run( fetch_package_data( fetch_npm_registry_metadata, current_app.config["NPM_CLIENT"], package_names, ), debug=False, ) if len(npm_registry_entries) != len(package_names): log.warn( f"only fetched {len(npm_registry_entries)} registry entries for {len(package_names)} package names" ) else: log.info( f"fetched {len(npm_registry_entries)} registry entries for {len(package_names)} package names" ) if current_app.config["NPM_CLIENT"].get("save_to_db", False): models.save_json_results(npm_registry_entries) # inserts new entries for new versions (but doesn't update old ones) models.insert_npm_registry_entries( serializers.serialize_npm_registry_entries( registry_entry for registry_entry in npm_registry_entries if registry_entry is not None)) return npm_registry_entries
def fetch_and_save_npmsio_scores(package_names: Iterable[str]) -> List[Dict]: package_names = list(package_names) log.info(f"fetching npmsio scores for {len(package_names)} package names") log.debug( f"fetching npmsio scores for package names: {list(package_names)}") npmsio_scores: List[Dict] = asyncio.run( fetch_package_data( fetch_npmsio_scores, current_app.config["NPMSIO_CLIENT"], package_names, ), debug=False, ) if len(npmsio_scores) != len(package_names): log.warn( f"only fetched {len(npmsio_scores)} scores for {len(package_names)} package names" ) else: log.info( f"fetched {len(npmsio_scores)} scores for {len(package_names)} package names" ) if current_app.config["NPMSIO_CLIENT"].get("save_to_db", False): models.save_json_results(npmsio_scores) models.insert_npmsio_scores( serializers.serialize_npmsio_scores(score for score in npmsio_scores if score is not None)) return npmsio_scores
def get_maintainer_breaches(package_name: str, package_version: str = None) -> None: registry_entries = get_NPMRegistryEntry(package_name).all() if not registry_entries: return registry_entry = registry_entries[0] if package_version: package_version_validation_error = ( validators.get_npm_package_version_validation_error(package_version) ) if package_version_validation_error is not None: raise package_version_validation_error for entry in registry_entries: if entry.package_version == package_version: registry_entry = entry break maintainers = registry_entry.maintainers emails = [maintainer["email"] for maintainer in maintainers] breach_results = dict() total_breaches = 0 if maintainers: breaches = fetch_breaches(emails) for email, breach_list in zip(emails, breaches): breach_results[email] = { "breach_num": len(breach_list), "breaches": breach_list, } total_breaches += len(breach_list) average_breaches = total_breaches / len(emails) if len(emails) else 0.0 result = { "package_name": package_name, "package_version": package_version, "breaches": breach_results, "total_breaches": total_breaches, "average_breaches": average_breaches, } save_json_results([result])
def get_github_advisories(package_name: str) -> None: github_client = current_app.config["GITHUB_CLIENT"] base_url = github_client["base_url"] github_auth_token = github_client["github_auth_token"] headers = {"Authorization": "token " + github_auth_token} query = (""" { securityVulnerabilities(ecosystem: NPM, first: 100, package: \"""" + package_name + """\", orderBy: {field: UPDATED_AT, direction: DESC}) { nodes { advisory { id, description, permalink, publishedAt, severity, summary, updatedAt, withdrawnAt } package { name } } pageInfo { endCursor, hasNextPage, hasPreviousPage, startCursor } totalCount } } """) response = requests.post(base_url, json={"query": query}, headers=headers) response.raise_for_status() nodes = response.json()["data"]["securityVulnerabilities"]["nodes"] advisories = list() ids = list() for node in nodes: if (node["advisory"]["id"] not in ids and node["advisory"]["withdrawnAt"] == None): advisory = node["advisory"] advisory["package"] = package_name advisories.append(advisory) ids.append(node["advisory"]["id"]) save_json_results(advisories)