def save_pubsub_message(app: flask.Flask,
message: gcp.pubsub_v1.types.PubsubMessage) -> None:
"""
Saves a pubsub message data to the JSONResult table and acks it.
nacks it if saving fails.
Requires depobs flask app context.
"""
with app.app_context():
try:
# TODO: set job status when it finishes? No, do this in the runner.
log.info(
f"received pubsub message {message.message_id} published at {message.publish_time} with attrs {message.attributes}"
)
save_json_results([{
"type":
"google.cloud.pubsub_v1.types.PubsubMessage",
"id":
message.message_id,
"publish_time":
flask.json.dumps(message.publish_time), # convert datetime
"attributes":
dict(message.attributes), # convert from ScalarMapContainer
"data":
flask.json.loads(message.data),
"size":
message.size,
}])
message.ack()
except Exception as err:
message.nack()
log.error(
f"error saving pubsub message {message} to json results table: {err}"
)
def fetch_and_save_registry_entries(
package_names: Iterable[str]) -> List[Dict]:
package_names = list(package_names)
log.info(
f"fetching registry entries for {len(package_names)} package names")
log.debug(
f"fetching registry entries for package names: {list(package_names)}")
npm_registry_entries = asyncio.run(
fetch_package_data(
fetch_npm_registry_metadata,
current_app.config["NPM_CLIENT"],
package_names,
),
debug=False,
)
if len(npm_registry_entries) != len(package_names):
log.warn(
f"only fetched {len(npm_registry_entries)} registry entries for {len(package_names)} package names"
)
else:
log.info(
f"fetched {len(npm_registry_entries)} registry entries for {len(package_names)} package names"
)
if current_app.config["NPM_CLIENT"].get("save_to_db", False):
models.save_json_results(npm_registry_entries)
# inserts new entries for new versions (but doesn't update old ones)
models.insert_npm_registry_entries(
serializers.serialize_npm_registry_entries(
registry_entry for registry_entry in npm_registry_entries
if registry_entry is not None))
return npm_registry_entries
def fetch_and_save_npmsio_scores(package_names: Iterable[str]) -> List[Dict]:
package_names = list(package_names)
log.info(f"fetching npmsio scores for {len(package_names)} package names")
log.debug(
f"fetching npmsio scores for package names: {list(package_names)}")
npmsio_scores: List[Dict] = asyncio.run(
fetch_package_data(
fetch_npmsio_scores,
current_app.config["NPMSIO_CLIENT"],
package_names,
),
debug=False,
)
if len(npmsio_scores) != len(package_names):
log.warn(
f"only fetched {len(npmsio_scores)} scores for {len(package_names)} package names"
)
else:
log.info(
f"fetched {len(npmsio_scores)} scores for {len(package_names)} package names"
)
if current_app.config["NPMSIO_CLIENT"].get("save_to_db", False):
models.save_json_results(npmsio_scores)
models.insert_npmsio_scores(
serializers.serialize_npmsio_scores(score for score in npmsio_scores
if score is not None))
return npmsio_scores
def get_maintainer_breaches(package_name: str, package_version: str = None) -> None:
registry_entries = get_NPMRegistryEntry(package_name).all()
if not registry_entries:
return
registry_entry = registry_entries[0]
if package_version:
package_version_validation_error = (
validators.get_npm_package_version_validation_error(package_version)
)
if package_version_validation_error is not None:
raise package_version_validation_error
for entry in registry_entries:
if entry.package_version == package_version:
registry_entry = entry
break
maintainers = registry_entry.maintainers
emails = [maintainer["email"] for maintainer in maintainers]
breach_results = dict()
total_breaches = 0
if maintainers:
breaches = fetch_breaches(emails)
for email, breach_list in zip(emails, breaches):
breach_results[email] = {
"breach_num": len(breach_list),
"breaches": breach_list,
}
total_breaches += len(breach_list)
average_breaches = total_breaches / len(emails) if len(emails) else 0.0
result = {
"package_name": package_name,
"package_version": package_version,
"breaches": breach_results,
"total_breaches": total_breaches,
"average_breaches": average_breaches,
}
save_json_results([result])
def get_github_advisories(package_name: str) -> None:
github_client = current_app.config["GITHUB_CLIENT"]
base_url = github_client["base_url"]
github_auth_token = github_client["github_auth_token"]
headers = {"Authorization": "token " + github_auth_token}
query = ("""
{
securityVulnerabilities(ecosystem: NPM, first: 100, package: \"""" +
package_name +
"""\", orderBy: {field: UPDATED_AT, direction: DESC}) {
nodes {
advisory {
id, description, permalink, publishedAt, severity, summary, updatedAt, withdrawnAt
}
package {
name
}
}
pageInfo {
endCursor, hasNextPage, hasPreviousPage, startCursor
}
totalCount
}
}
""")
response = requests.post(base_url, json={"query": query}, headers=headers)
response.raise_for_status()
nodes = response.json()["data"]["securityVulnerabilities"]["nodes"]
advisories = list()
ids = list()
for node in nodes:
if (node["advisory"]["id"] not in ids
and node["advisory"]["withdrawnAt"] == None):
advisory = node["advisory"]
advisory["package"] = package_name
advisories.append(advisory)
ids.append(node["advisory"]["id"])
save_json_results(advisories)