def construct_package_query(cls, input_json):
"""Construct the query to retrieve detailed information of given package."""
# TODO: reduce cyclomatic complexity
# see https://fabric8-analytics.github.io/dashboard/fabric8-analytics-data-model.cc.D.html
# issue: https://github.com/fabric8-analytics/fabric8-analytics-data-model/issues/232
pkg_name = input_json.get('package')
ecosystem = input_json.get('ecosystem')
pkg_name_tokens = re.split(r'\W+', pkg_name)
prp_package = ""
drop_prop = ""
drop_props = []
# TODO: refactor into the separate module
str_package = "pkg = g.V().has('ecosystem','{ecosystem}').has('name', '{pkg_name}')." \
"tryNext().orElseGet{{g.V()." \
"has('vertex_label','Count').choose(has('{ecosystem}_pkg_count')," \
"sack(assign).by('{ecosystem}_pkg_count').sack(sum).by(constant(" \
"1)).property('{ecosystem}_pkg_count',sack())," \
"property('{ecosystem}_pkg_count',1)).iterate();" \
"graph.addVertex('ecosystem', '{ecosystem}', 'name', " \
"'{pkg_name}', 'vertex_label', 'Package'); }};" \
"pkg.property('last_updated', {last_updated});".format(
ecosystem=ecosystem, pkg_name=pkg_name, last_updated=str(time.time()))
cur_latest_ver, cur_libio_latest_ver = get_current_version(ecosystem, pkg_name)
cur_date = (datetime.utcnow()).strftime('%Y%m%d')
latest_version = cls.sanitize_text_for_query(input_json.get('latest_version'))
if latest_version:
# If latest version dont have cve, then it becomes the latest non cve version as well
non_cve_ver = get_latest_version_non_cve(ecosystem, pkg_name, latest_version)
prp_package += "pkg.property('latest_non_cve_version', '{}');".format(non_cve_ver)
prp_package += "pkg.property('latest_version', '{}');".format(latest_version)
if latest_version != cur_latest_ver:
prp_package += "pkg.property('latest_version_last_updated', '{}');".format(cur_date)
# Get Github Details
if 'github_details' in input_json.get('analyses', {}):
gh_details = input_json.get('analyses', {})\
.get('github_details', {}).get('details', {})
prp_package += create_query("prs",
gh_details.get("updated_pull_requests", {}), "month")
prp_package += create_query("prs",
gh_details.get("updated_pull_requests", {}), "year")
prp_package += create_query("issues", gh_details.get("updated_issues", {}), "month")
prp_package += create_query("issues", gh_details.get("updated_issues", {}), "year")
prp_package += set_property(gh_details, 'forks_count', 'gh_forks')
prp_package += set_property(gh_details, 'stargazers_count', 'gh_stargazers')
prp_package += set_property(gh_details, 'open_issues_count', 'gh_open_issues_count')
prp_package += set_property(gh_details, 'subscribers_count', 'gh_subscribers_count')
prp_package += set_property(gh_details, 'subscribers_count', 'gh_subscribers_count')
prp_package += set_property(gh_details, 'contributors_count', 'gh_contributors_count')
gh_refreshed_on = gh_details.get('updated_on')
if gh_refreshed_on:
prp_package += "pkg.property('gh_refreshed_on', '{}');".format(gh_refreshed_on)
# Add github topics
topics = gh_details.get('topics', [])
if topics:
drop_props.append('topics')
str_package += " ".join(["pkg.property('topics', '{}');".format(t)
for t in topics if t])
# Add tokens for a package
if pkg_name_tokens:
drop_props.append('tokens')
str_package += " ".join(["pkg.property('tokens', '{}');".format(t)
for t in pkg_name_tokens if t])
# Refresh the properties whereever applicable
if len(drop_props) > 0:
drop_prop += "g.V().has('ecosystem','{ecosystem}').has('name'," \
"'{pkg_name}').properties('{p}').drop().iterate();".format(
ecosystem=ecosystem, pkg_name=pkg_name, p="','".join(drop_props))
return drop_prop + str_package, prp_package
def construct_package_query(cls, input_json):
"""Construct the query to retrieve detailed information of given package."""
# TODO: reduce cyclomatic complexity
# see https://fabric8-analytics.github.io/dashboard/fabric8-analytics-data-model.cc.D.html
# issue: https://github.com/fabric8-analytics/fabric8-analytics-data-model/issues/232
pkg_name = input_json.get('package')
ecosystem = input_json.get('ecosystem')
pkg_name_tokens = re.split(r'\W+', pkg_name)
prp_package = ""
drop_prop = ""
drop_props = []
# TODO: refactor into the separate module
str_package = "pkg = g.V().has('ecosystem','{ecosystem}').has('name', '{pkg_name}')." \
"tryNext().orElseGet{{g.V()." \
"has('vertex_label','Count').choose(has('{ecosystem}_pkg_count')," \
"sack(assign).by('{ecosystem}_pkg_count').sack(sum).by(constant(" \
"1)).property('{ecosystem}_pkg_count',sack())," \
"property('{ecosystem}_pkg_count',1)).iterate();" \
"graph.addVertex('ecosystem', '{ecosystem}', 'name', " \
"'{pkg_name}', 'vertex_label', 'Package'); }};" \
"pkg.property('last_updated', {last_updated});".format(
ecosystem=ecosystem, pkg_name=pkg_name, last_updated=str(time.time()))
cur_latest_ver, cur_libio_latest_ver = get_current_version(
ecosystem, pkg_name)
cur_date = (datetime.utcnow()).strftime('%Y%m%d')
last_updated_flag = 'false'
latest_version = cls.sanitize_text_for_query(
input_json.get('latest_version'))
if latest_version:
# If latest version dont have cve, then it becomes the latest non cve version as well
non_cve_ver = get_latest_version_non_cve(ecosystem, pkg_name,
latest_version)
prp_package += "pkg.property('latest_non_cve_version', '{}');".format(
non_cve_ver)
prp_package += "pkg.property('latest_version', '{}');".format(
latest_version)
if latest_version != cur_latest_ver:
prp_package += "pkg.property('latest_version_last_updated', '{}');".format(
cur_date)
last_updated_flag = 'true'
# Get Github Details
if 'github_details' in input_json.get('analyses', {}):
gh_details = input_json.get('analyses', {})\
.get('github_details', {}).get('details', {})
prp_package += create_query(
"prs", gh_details.get("updated_pull_requests", {}), "month")
prp_package += create_query(
"prs", gh_details.get("updated_pull_requests", {}), "year")
prp_package += create_query("issues",
gh_details.get("updated_issues", {}),
"month")
prp_package += create_query("issues",
gh_details.get("updated_issues", {}),
"year")
prp_package += set_property(gh_details, 'forks_count', 'gh_forks')
prp_package += set_property(gh_details, 'stargazers_count',
'gh_stargazers')
prp_package += set_property(gh_details, 'open_issues_count',
'gh_open_issues_count')
prp_package += set_property(gh_details, 'subscribers_count',
'gh_subscribers_count')
prp_package += set_property(gh_details, 'subscribers_count',
'gh_subscribers_count')
prp_package += set_property(gh_details, 'contributors_count',
'gh_contributors_count')
gh_refreshed_on = gh_details.get('updated_on')
if gh_refreshed_on:
prp_package += "pkg.property('gh_refreshed_on', '{}');".format(
gh_refreshed_on)
# Add github topics
topics = gh_details.get('topics', [])
if topics:
drop_props.append('topics')
str_package += " ".join([
"pkg.property('topics', '{}');".format(t) for t in topics
if t
])
# Add tokens for a package
if pkg_name_tokens:
drop_props.append('tokens')
str_package += " ".join([
"pkg.property('tokens', '{}');".format(t)
for t in pkg_name_tokens if t
])
# Get Libraries.io data
if 'libraries_io' in input_json.get('analyses', {}):
v2 = input_json['analyses']['libraries_io'].get('schema', {}).get('version', '0-0-0') \
>= '2-0-0'
details = input_json['analyses']['libraries_io'].get('details', {})
libio_dependents_projects = details.get('dependents',
{}).get('count', -1)
libio_dependents_repos = details.get('dependent_repositories',
{}).get('count', -1)
releases = details.get('releases', {})
libio_total_releases = int(releases.get('count', -1))
libio_latest_version = libio_latest_published_at = ''
if libio_total_releases > 0:
if v2:
libio_latest = releases.get('recent',
[{}])[-1] # last is latest
libio_latest_published_at = libio_latest.get(
'published_at', '')
libio_latest_version = libio_latest.get('number', '')
else:
libio_latest_published_at = releases.get('latest', {}).get(
'published_at', '')
libio_latest_version = releases.get('latest',
{}).get('version', '')
if libio_latest_version != cur_libio_latest_ver and last_updated_flag != 'true':
prp_package += "pkg.property('latest_version_last_updated', '{}');" \
.format(cur_date)
if libio_latest_published_at:
t = libio_latest_published_at
p = parse_datetime(t).timetuple() if t else ''
published_at = str(time.mktime(p)) if p else ''
prp_package += "pkg.property('libio_latest_release', '{}');".format(
published_at)
if details.get('dependent_repositories', {}).get('top'):
drop_props.append('libio_usedby')
for key, val in details.get('dependent_repositories',
{}).get('top', {}).items():
prp_package += "pkg.property('libio_usedby', '{key}:{val}');".format(
key=key, val=val)
prp_package += "pkg.property('libio_dependents_projects', " \
"'{libio_dependents_projects}');" \
"pkg.property('libio_dependents_repos', '{libio_dependents_repos}');" \
"pkg.property('libio_total_releases', '{libio_total_releases}');" \
"pkg.property('libio_latest_version', '{libio_latest_version}');".format(
libio_dependents_projects=libio_dependents_projects,
libio_dependents_repos=libio_dependents_repos,
libio_total_releases=libio_total_releases,
libio_latest_version=libio_latest_version)
# Update EPV Github Release Date based on libraries_io data
if v2:
# 'recent' is list of {'number':n, 'published_at':p} including the latest
for release in releases.get('recent', []):
rel_published = release.get('published_at', '')
parsed_dt = parse_datetime(
rel_published).timetuple() if rel_published else ''
timestamp = time.mktime(parsed_dt) if parsed_dt else ''
prp_package += "g.V().has('pecosystem','{ecosystem}').has('pname'," \
"'{pkg_name}').has('version','{version}')." \
"property('gh_release_date',{gh_rel});".format(
ecosystem=ecosystem, pkg_name=pkg_name,
version=release.get('number', ''),
gh_rel=str(timestamp))
else:
if libio_latest_published_at:
gh_release = time.mktime(
parse_datetime(libio_latest_published_at).timetuple())
prp_package += "g.V().has('pecosystem','{ecosystem}').has('pname'," \
"'{pkg_name}')." \
"has('version','{libio_latest_version}')." \
"property('gh_release_date', {gh_rel});".format(
pkg_name=pkg_name, ecosystem=ecosystem,
libio_latest_version=libio_latest_version,
gh_rel=str(gh_release))
for version, release in releases.get('latest',
{}).get('recent',
{}).items():
prp_package += "g.V().has('pecosystem','{ecosystem}').has('pname'," \
"'{pkg_name}').has('version','{version}')." \
"property('gh_release_date',{gh_rel});".format(
ecosystem=ecosystem, pkg_name=pkg_name, version=version,
gh_rel=str(time.mktime(parse_datetime(release).timetuple())))
# Refresh the properties whereever applicable
if len(drop_props) > 0:
drop_prop += "g.V().has('ecosystem','{ecosystem}').has('name'," \
"'{pkg_name}').properties('{p}').drop().iterate();".format(
ecosystem=ecosystem, pkg_name=pkg_name, p="','".join(drop_props))
return drop_prop + str_package, prp_package
def test_get_current_version():
"""Test the function get_version_information."""
out1, out2 = get_current_version('maven', 'io.vertx:vertx-web')
assert out1 == -1
assert out2 == -1
