def test_add_license_attr():
p = Package.load_from_json(serve_static_json)
assert p.save() is not None
v = Version.load_from_json(serve_static_json, package=p)
v.save()
p.create_version_edge(v)
ts1 = v.last_updated
assert p.last_updated is not None
assert ts1 is not None
license_data = serve_static_json["analyses"]["source_licenses"]
_, _, licenses = LicenseDetails.load_from_json(license_data)
v.add_license_attribute(licenses)
assert v.last_updated >= ts1
version_criteria = {
'pecosystem': v.ecosystem_package.ecosystem,
'pname': v.ecosystem_package.name,
'version': v.version
}
present_version = Version.find_by_criteria('Version', p, version_criteria)
assert present_version.last_updated == v.last_updated
assert (len(present_version.licenses) == 1)
test_set = ['MITNFA']
assert present_version.licenses == test_set
Version.delete_by_id(v.id)
Package.delete_by_id(p.id)
def test_empty_github_results():
p = Package.load_from_json(npm_sequence_3)
assert p.save() is not None
v = Version.load_from_json(npm_sequence_3, package=p)
v.save()
p.create_version_edge(v)
assert p.last_updated is not None
assert v.last_updated is not None
# input_json = gv.read_from_file('test/data/npm-sequence-3.0.0.json')
github_data = npm_sequence_3["analyses"]["github_details"]
github_result = GithubResult.load_from_json(github_data)
assert (github_result.details is not None)
assert github_result.last_updated is None
# gid = github_result.save()
v.add_edge_github_details(github_result)
ls_before = github_result.last_updated
assert ls_before is not None
assert (github_result.id is not None)
v.add_edge_github_details(github_result)
ls_after = github_result.last_updated
assert ls_after >= ls_before
assert GithubResult.count() == 1
GithubResult.delete_by_id(github_result.id)
Version.delete_by_id(v.id)
Package.delete_by_id(p.id)
def test_person_author():
p = Package.load_from_json(serve_static_json)
assert p.save() is not None
v = Version.load_from_json(serve_static_json, package=p)
v.save()
p.create_version_edge(v)
assert p.last_updated is not None
assert v.last_updated is not None
authors_data = serve_static_json["analyses"]["metadata"]
authors_list = Author.load_from_json(authors_data)
ts_list = []
for a in authors_list:
a.save()
assert a.last_updated is not None
ts_list.append(a.last_updated)
v.add_edge_author(a)
author_before = Author.count()
assert (author_before == 1)
author_detail = Author(name='Douglas Christopher Wilson',
email='*****@*****.**')
author_detail.save()
assert author_detail.last_updated >= a.last_updated
assert (Author.count() == 1)
for a in authors_list:
Author.delete_by_id(a.id)
Author.delete_by_id(author_detail.id)
Version.delete_by_id(v.id)
Package.delete_by_id(p.id)
def test_person_contributor():
p = Package.load_from_json(serve_static_json)
assert p.save() is not None
v = Version.load_from_json(serve_static_json, package=p)
v.save()
p.create_version_edge(v)
assert p.last_updated is not None
assert v.last_updated is not None
contributors_data = serve_static_json["analyses"]["metadata"]
contributors_list = Contributor.load_from_json(contributors_data)
for c in contributors_list:
c.save()
assert c.last_updated is not None
v.add_edge_contributor(c)
assert (Contributor.count() == 0)
for c in contributors_list:
Contributor.delete_by_id(c.id)
Version.delete_by_id(v.id)
Package.delete_by_id(p.id)
def test_support_vector_github_detail():
p = Package.load_from_json(serve_static_json)
assert p.save() is not None
v = Version.load_from_json(serve_static_json, package=p)
v.save()
p.create_version_edge(v)
assert p.last_updated is not None
assert v.last_updated is not None
github_data = serve_static_json["analyses"]["github_details"]
github_result = GithubResult.load_from_json(github_data)
assert github_result.last_updated is None
v.add_edge_github_details(github_result)
ls_before = github_result.last_updated
assert (GithubResult.count() == 1)
count_before = len(v.get_version_out_edge('has_github_details'))
assert count_before == 1
#try adding the edge again
v.add_edge_github_details(github_result)
count_after = len(v.get_version_out_edge('has_github_details'))
ls_after = github_result.last_updated
assert count_before == count_after
assert ls_after >= ls_before
GithubResult.delete_by_id(github_result.id)
Version.delete_by_id(v.id)
Package.delete_by_id(p.id)
def test_blackduck_graph():
bl_list = []
p = Package.load_from_json(serve_static_json)
assert p.save() is not None
v = Version.load_from_json(serve_static_json, package=p)
v.save()
p.create_version_edge(v)
assert p.last_updated is not None
assert v.last_updated is not None
bl_list = bl.load_from_json(input_json['analyses'])
assert len(bl_list) == 1
objBlackduck = bl.add_blackduck_issue(bl_list[0])
v.add_blackduck_cve_edge(objBlackduck.id)
bl_criteria = {'vulnerability_name': 'CVE-2015-1164'}
obj_fetch = SecurityDetails.find_by_criteria('CVE', bl_criteria)
assert obj_fetch.last_updated == objBlackduck.last_updated
SecurityDetails.delete_by_id(obj_fetch.id)
Version.delete_by_id(v.id)
Package.delete_by_id(p.id)
def load_dependencies(ecosystem, dependency_data):
dependency_pck_list = []
dependency_ver_list = []
dependency_type = []
dependencies = get_dependencies(dependency_data)
for dep_type, dep_list in dependencies.items():
for d in dep_list:
splits = d.split(" ")
(n, v) = ("", "")
if len(splits) >= 1:
n = splits[0]
if len(splits) >= 2:
v = splits[1]
pck_criteria_dict = {'ecosystem': ecosystem, 'name': n}
pck_dep = Package.find_by_criteria(
'Package', pck_criteria_dict) or Package(ecosystem, n)
ver_criteria_dict = {
'pecosystem': ecosystem,
'pname': n,
'version': v
}
ver_dep = Version.find_by_criteria(
'Version', pck_dep, ver_criteria_dict) or Version(pck_dep, v)
dependency_pck_list.append(pck_dep)
dependency_ver_list.append(ver_dep)
dependency_type.append(dep_type)
return dependency_pck_list, dependency_ver_list, dependency_type
def test_support_vector_security():
p = Package.load_from_json(serve_static_json)
assert p.save() is not None
v = Version.load_from_json(serve_static_json, package=p)
v.save()
p.create_version_edge(v)
assert p.last_updated is not None
assert v.last_updated is not None
security_data = serve_static_json["analyses"]["security_issues"]
security_list, cvss_score, cve_ids = SecurityDetails.load_from_json(
security_data)
ts_list = []
for s, cvss, cve in zip(security_list, cvss_score, cve_ids):
s.save()
assert s.last_updated is not None
ts_list.append(s.last_updated)
v.add_security_edge(s, cvss)
security_before = SecurityDetails.count()
assert (security_before == 1)
present_security = SecurityDetails.find_by_criteria(
'CVE', {'cve_id': 'CVE-2015-1164'})
assert (len(present_security.references) == 5)
ref_list = [
"https://github.com/expressjs/serve-static/issues/26",
"https://bugzilla.redhat.com/show_bug.cgi?id=1181917",
"http://xforce.iss.net/xforce/xfdb/99936",
"http://www.securityfocus.com/bid/72064",
"http://nodesecurity.io/advisories/serve-static-open-redirect"
]
assert (all(r in ref_list for r in present_security.references))
repeat_security_detail = SecurityDetails(cve_id='CVE-2015-1164',
cvss=4.3,
summary='')
repeat_security_detail.issue_has_access('authentication', '')
repeat_security_detail.issue_has_access('vector', 'NETWORK')
repeat_security_detail.issue_has_access('complexity', 'MEDIUM')
repeat_security_detail.issue_has_impact('integrity', 'partial')
repeat_security_detail.issue_has_impact('confidentiality', '')
repeat_security_detail.issue_has_impact('availability', '')
repeat_security_detail.save()
assert repeat_security_detail.id == s.id
assert repeat_security_detail.last_updated >= ts_list[0]
assert (SecurityDetails.count() == 1)
for s in security_list:
SecurityDetails.delete_by_id(s.id)
SecurityDetails.delete_by_id(repeat_security_detail.id)
Version.delete_by_id(v.id)
Package.delete_by_id(p.id)
def test_additional_data_as_attr():
add_details = {}
pck_obj = Package.load_from_json(serve_static_json)
assert pck_obj.last_updated is None
assert pck_obj.save() is not None
ver_obj = Version.load_from_json(serve_static_json, package=pck_obj)
assert ver_obj.last_updated is None
ver_obj.save()
v_ts1 = ver_obj.last_updated
assert pck_obj.last_updated is not None
assert v_ts1 is not None
pck_obj.create_version_edge(ver_obj)
code_metrics = CMR.load_from_json(
npm_crumb_data["analyses"]["code_metrics"])
security_data = serve_static_json["analyses"]["security_issues"]
security_list, cvss_score, cve_ids = SD.load_from_json(security_data)
if len(cvss_score) > 0 and len(cve_ids) > 0:
cve_ids = cve_ids
cvss = cvss_score
add_details["cve_ids"] = cve_ids[0]
add_details["cvss"] = cvss[0]
ver_obj.add_additional_data_as_attr(code_metrics)
ver_obj.add_cve_ids(add_details["cvss"], add_details["cve_ids"])
assert ver_obj.last_updated >= v_ts1
version_criteria = {
'pecosystem': pck_obj.ecosystem,
'pname': pck_obj.name,
'version': ver_obj.version
}
present_version = Version.find_by_criteria(ver_obj.label, pck_obj,
version_criteria)
logger.info(present_version.__dict__)
assert present_version.cm_loc == 1351
assert present_version.cm_num_files == 16
assert present_version.cm_avg_cyclomatic_complexity == -1
assert present_version.cve_ids == ['CVE-2015-1164:7.5']
assert present_version.last_updated == ver_obj.last_updated
Version.delete_by_id(ver_obj.id)
Package.delete_by_id(pck_obj.id)
def test_create_package_entity():
packages = Package.find_all()
assert (len(packages) == 0)
list_epv_1 = [{
'ecosystem': 'maven',
'name': 'org.slf4j:slf4j-api',
'version': '1.5.6'
}]
import_epv_from_folder('test/data/S3-data', list_epv=list_epv_1)
criteria_dict = {'ecosystem': 'maven', 'name': 'org.slf4j:slf4j-api'}
p = Package.find_by_criteria('Package', criteria_dict)
assert p.latest_version == '1.7.22'
p.save() # must be an update
assert (Package.count() == 2)
p.create() # duplicate should not create new node
assert (Package.count() == 2)
criteria_dict = {'ecosystem': 'maven', 'name': 'junit:junit'}
p2 = Package.find_by_criteria('Package', criteria_dict)
assert p2.latest_version == '-1'
list_epv_2 = [{
'ecosystem': 'maven',
'name': 'junit:junit',
'version': '4.8.2'
}]
import_epv_from_folder('test/data/S3-data', list_epv=list_epv_2)
criteria_dict = {'ecosystem': 'maven', 'name': 'junit:junit'}
p3 = Package.find_by_criteria('Package', criteria_dict)
assert p3.latest_version == '4.12'
p.save() # must be an update
assert (Package.count() == 2)
Package.delete_all()
assert (Package.count() == 0)
Version.delete_all()
assert (Version.count() == 0)
def test_package_import_from_folder():
package_dir = 'test/data/S3-data/maven/junit:junit'
packages = Package.find_all()
assert (len(packages) == 0)
versions = Version.find_all()
assert (len(versions) == 0)
p = Package()
assert (p.id is None)
assert (p.label == "Package")
assert p.last_updated is None
report = import_from_folder(package_dir)
assert (report.get('status') == 'Success')
assert (report.get('count_imported_EPVs') == 1)
assert (report.get('last_imported_EPV') == '4.8.2.json')
assert (report.get('max_finished_at') == '2017-02-24T13:42:29.665786')
criteria_dict = {'ecosystem': 'maven', 'name': 'junit:junit'}
p1 = Package.find_by_criteria('Package', criteria_dict)
assert p1 is not None
assert p1.id is not None
assert (p1.ecosystem == 'maven')
assert (p1.latest_version == '4.12')
assert (p1.package_dependents_count == -1)
assert (p1.name == 'junit:junit')
criteria_dict = {
'pecosystem': 'maven',
'pname': 'junit:junit',
'version': '4.8.2'
}
v1 = Version.find_by_criteria('Version', p1, criteria_dict)
assert v1 is not None
assert v1.ecosystem_package is not None
assert (v1.ecosystem_package.ecosystem == 'maven')
packages = Package.find_all()
assert (len(packages) == 1)
versions = Version.find_all()
assert (len(versions) == 1)
Package.delete_by_id(p1.id)
assert (Package.count() == 0)
Version.delete_by_id(v1.id)
assert (Version.count() == 0)
GraphMetaData.delete_all()
assert (GraphMetaData.count() == 0)
def test_version_dependencies():
p = Package.load_from_json(serve_static_json)
assert p.save() is not None
v = Version.load_from_json(serve_static_json, package=p)
v.save()
p.create_version_edge(v)
dependency_data = serve_static_json["analyses"]["dependency_snapshot"]
dependency_pck_list, dependency_ver_list, dependency_type = \
vdv.load_dependencies(v.ecosystem_package.ecosystem, dependency_data)
for d_pck, d_ver, d_type in zip(dependency_pck_list, dependency_ver_list,
dependency_type):
assert d_pck.save() is not None
d_ver.save()
d_pck.create_version_edge(d_ver)
v.add_edge_dependency(d_ver, d_type)
assert (Version.count_dependency(v.id) == 4)
assert (Version.count() + Package.count() == 10)
for pd, vd in zip(dependency_pck_list, dependency_ver_list):
Version.delete_by_id(vd.id)
Package.delete_by_id(pd.id)
Version.delete_by_id(v.id)
Package.delete_by_id(p.id)
def test_github_attr():
pck_obj = Package.load_from_json(serve_static_json)
assert pck_obj.last_updated is None
assert pck_obj.save() is not None
p_ts1 = pck_obj.last_updated
assert p_ts1 is not None
assert Package.count() == 1
github_result = load_github_result_from_json(serve_static_json)
assert github_result.last_updated is None
pck_obj.add_github_details_as_attr(github_result)
assert pck_obj.last_updated >= p_ts1
package_criteria = {'ecosystem': pck_obj.ecosystem, 'name': pck_obj.name}
present_package = Package.find_by_criteria('Package', package_criteria)
assert present_package.last_updated == pck_obj.last_updated
assert present_package.gh_forks == 84
assert present_package.gh_stargazers == 538
assert present_package.gh_issues_last_year_opened == 15
assert present_package.gh_issues_last_year_closed == 16
assert present_package.gh_issues_last_month_opened == 0
assert present_package.gh_issues_last_month_closed == 0
assert present_package.gh_prs_last_year_opened == 11
assert present_package.gh_prs_last_year_closed == 11
assert present_package.gh_prs_last_month_opened == 1
assert present_package.gh_prs_last_month_closed == 1
Package.delete_by_id(pck_obj.id)
assert Package.count() == 0
def test_add_code_metrics_non_empty():
p = Package.load_from_json(npm_crumb_data)
assert p.save() is not None
v = Version.load_from_json(npm_crumb_data, package=p)
v.save()
p.create_version_edge(v)
assert p.last_updated is not None
assert v.last_updated is not None
code_metrics_data = npm_crumb_data["analyses"]["code_metrics"]
code_metrics = CodeMetricsResult.load_from_json(code_metrics_data)
assert code_metrics.last_updated is None
assert code_metrics.id is None
v.add_code_metrics_edge(code_metrics)
assert code_metrics.id >= 0
assert code_metrics.last_updated is not None
assert code_metrics.last_updated > v.last_updated
assert (CodeMetricsResult.count() == 1)
assert (CodeMetricsLanguage.count() == 6)
count_before = len(v.get_version_out_edge('has_code_metrics'))
assert count_before == 1
#try adding the edge again, should not let this happen
results2 = v.add_code_metrics_edge(code_metrics)
count_after = len(v.get_version_out_edge('has_code_metrics'))
assert count_after == count_before
assert CodeMetricsLanguage.count() == 6
lang_nodes = CodeMetricsLanguage.find_all()
for lang_node in lang_nodes:
CodeMetricsLanguage.delete_by_id(lang_node.id)
CodeMetricsLanguage.delete_by_id(code_metrics.id)
Version.delete_by_id(v.id)
Package.delete_by_id(p.id)
def test_bucket_import_from_folder():
bucket_dir = 'test/data/S3-data'
packages = Package.find_all()
assert (len(packages) == 0)
p = Package()
assert (p.id is None)
assert (p.label == "Package")
assert p.last_updated is None
report = import_from_folder(bucket_dir)
assert (report.get('status') == 'Success')
assert (report.get('count_imported_EPVs') == 3)
assert (report.get('last_imported_EPV') == 'npm/sequence/3.0.0.json')
assert (report.get('max_finished_at') == '2017-02-24T13:43:11.872916')
criteria_dict = {'ecosystem': 'npm', 'name': 'sequence'}
p1 = Package.find_by_criteria('Package', criteria_dict)
assert p1 is not None
assert p1.id is not None
assert (p1.ecosystem == 'npm')
assert (p1.latest_version == '3.0.0')
assert (p1.package_dependents_count == 22)
assert (p1.name == 'sequence')
criteria_dict = {
'pecosystem': 'npm',
'pname': 'sequence',
'version': '3.0.0'
}
v1 = Version.find_by_criteria('Version', p1, criteria_dict)
assert v1 is not None
assert v1.ecosystem_package is not None
assert (v1.ecosystem_package.ecosystem == 'npm')
packages = Package.find_all()
assert (len(packages) == 3)
versions = Version.find_all()
assert (len(versions) == 4)
Package.delete_all()
assert (Package.count() == 0)
Version.delete_all()
assert (Version.count() == 0)
GraphMetaData.delete_all()
assert (GraphMetaData.count() == 0)
def test_support_vector_license():
p = Package.load_from_json(serve_static_json)
assert p.save() is not None
v = Version.load_from_json(serve_static_json, package=p)
v.save()
p.create_version_edge(v)
assert p.last_updated is not None
assert v.last_updated is not None
license_data = serve_static_json["analyses"]["source_licenses"]
license_details_list, license_counts_list, _ = LicenseDetails.load_from_json(
license_data)
ts_list = []
for license_detail, license_count in zip(license_details_list,
license_counts_list):
license_detail.save()
assert license_detail.last_updated is not None
ts_list.append(license_detail.last_updated)
v.add_license_edge(license_detail, license_count)
assert (LicenseDetails.count() == 1)
new_license_detail = LicenseDetails(name='MITNFA')
new_license_detail.save()
assert new_license_detail.last_updated >= ts_list[0]
# Duplicate license should not be inserted
assert (LicenseDetails.count() == 1)
for l_id in license_details_list:
LicenseDetails.delete_by_id(l_id.id)
LicenseDetails.delete_by_id(new_license_detail.id)
Version.delete_by_id(v.id)
Package.delete_by_id(p.id)
def test_create_package_entity():
src_dir = 'test/data/full_import/npm/serve-static'
data_source = LocalFileSystem(src_dir)
list_keys = data_source.list_files()
grouped_keys = _group_keys_directory(list_keys, data_source.src_dir)
report = _import_grouped_keys_http(data_source, grouped_keys)
assert report is not None
assert report['status'] == 'Success'
assert report['message'] == 'The import finished successfully!'
assert report['last_imported_EPV'] == '1.7.1.json'
assert report['count_imported_EPVs'] == 1
assert report['max_finished_at'] == '2017-02-08T12:26:51.962609'
criteria_dict = {'ecosystem': 'npm', 'name': 'serve-static'}
p = Package.find_by_criteria('Package', criteria_dict)
assert p is not None
assert p.ecosystem == 'npm'
assert p.name == 'serve-static'
assert p.tokens == ['serve', 'static']
assert Package.count() == 1
Package.delete_by_id(p.id)
assert Package.count() == 0
def test_package() -> None:
package = Package(
'009e35ef-1f50-4bf3-ab58-11eb85bf5503',
Name('Soldered Wire Connector 1x19 ⌀1.0mm'),
Description(
'A 1x19 soldered wire connector with 2.54mm pin spacing and 1.0mm drill holes.\n\nGenerated with librepcb-parts-generator (generate_connectors.py)'
), Keywords('connector, 1x19, d1.0, connector, soldering, generic'),
Author('Danilo B.'), Version('0.1'), Created('2018-10-17T19:13:41Z'),
Deprecated(False), Category('56a5773f-eeb4-4b39-8cb9-274f3da26f4f'))
package.add_pad(
PackagePad('5c4d39d3-35cc-4836-a082-693143ee9135', Name('1')))
package.add_pad(
PackagePad('6100dd55-d3b3-4139-9085-d5a75e783c37', Name('2')))
package.add_footprint(create_footprint())
assert str(
package) == """(librepcb_package 009e35ef-1f50-4bf3-ab58-11eb85bf5503
def test_version_entity():
p = Package.load_from_json(serve_static_json)
assert p.save() is not None
v = Version.load_from_json(serve_static_json, package=p)
v.save()
assert (Version.count() == 1)
ls_before = v.last_updated
assert ls_before is not None
criteria_dict = {
'pecosystem': 'npm',
'pname': 'serve-static',
'version': '1.7.1'
}
v2 = Version.find_by_criteria('Version', p, criteria_dict)
assert v2.last_updated == v.last_updated
v.save()
ls_after = v.last_updated
assert (Version.count() == 1)
assert ls_after >= ls_before
assert v.last_updated >= v2.last_updated
test_packaged_in = ['nodejs-serve-static']
test_published_in = []
assert (all(pck in test_packaged_in for pck in v.is_packaged_in))
assert (all(pub in test_published_in for pck in v.is_published_in))
# now create an edge
edge_count_before = Package.edge_count()
p.create_version_edge(v)
edge_count_after = Package.edge_count()
assert (edge_count_after == edge_count_before + 1)
# now try to create an edge again
edge_count_before = Package.edge_count()
p.create_version_edge(v)
edge_count_after = Package.edge_count()
assert (edge_count_after == edge_count_before)
# this should return all versions associated with this package
p_versions = p.get_versions()
assert (len(p_versions) == 1)
Version.delete_by_id(v.id)
Package.delete_by_id(p.id)
def test_ecosystem_import_from_folder():
ecosystem_dir = 'test/data/S3-data/maven'
packages = Package.find_all()
assert (len(packages) == 0)
p = Package()
assert (p.id is None)
assert (p.label == "Package")
assert p.last_updated is None
report = import_from_folder(ecosystem_dir)
assert (report.get('status') == 'Success')
assert (report.get('count_imported_EPVs') == 2)
assert (
report.get('last_imported_EPV') == 'org.slf4j:slf4j-api/1.5.6.json')
assert (report.get('max_finished_at') == '2017-02-24T13:43:11.872916')
criteria_dict = {'ecosystem': 'maven', 'name': 'junit:junit'}
p1 = Package.find_by_criteria('Package', criteria_dict)
criteria_dict = {
'pecosystem': 'maven',
'pname': 'junit:junit',
'version': '4.8.2'
}
v1 = Version.find_by_criteria('Version', p1, criteria_dict)
assert v1 is not None
assert v1.ecosystem_package is not None
assert (v1.ecosystem_package.ecosystem == 'maven')
packages = Package.find_all()
assert (len(packages) == 2)
versions = Version.find_all()
assert (len(versions) == 3)
Package.delete_all()
assert (Package.count() == 0)
Version.delete_all()
assert (Version.count() == 0)
GraphMetaData.delete_all()
assert (GraphMetaData.count() == 0)
def load_package_version_values(filename):
input_json = gv.read_from_file(filename)
objpackage = Package.load_from_json(input_json)
objversion = Version.load_from_json(input_json, objpackage)
github_data = input_json["analyses"]["github_details"]
github_result = GithubResult.load_from_json(github_data)
objversion.github_details = github_result
authors_data = input_json["analyses"]["metadata"]
authors_list = Author.load_from_json(authors_data)
for objauthor in authors_list:
objversion.version_authored_by(objauthor)
contributors_data = input_json["analyses"]["metadata"]
contributor_list = Contributor.load_from_json(contributors_data)
for objcontributor in contributor_list:
objversion.version_contributed_by(objcontributor)
license_data = input_json["analyses"]["source_licenses"]
license_details_list, license_counts_list, license_names = LicenseDetails.load_from_json(
license_data)
for objlicense, license_count in zip(license_details_list,
license_counts_list):
objversion.version_covered_under(objlicense, license_count)
objversion.licenses = license_names
dependency_data = input_json["analyses"]["dependency_snapshot"]
_, dependency_ver_list, dependency_type = \
vdv.load_dependencies(
objversion.ecosystem_package.ecosystem, dependency_data)
for d_ver, d_type in zip(dependency_ver_list, dependency_type):
objversion.version_depends_on(d_ver, d_type)
security_data = input_json["analyses"]["security_issues"]
security_list, __, _ = SecurityDetails.load_from_json(security_data)
for objsecurity in security_list:
objversion.version_has_nvd_issues(objsecurity)
return objversion
def save(self):
package_criteria = {
'ecosystem': self.ecosystem_package.ecosystem,
'name': self.ecosystem_package.name
}
pck_obj = Package.find_by_criteria('Package', package_criteria)
if pck_obj is None:
logger.error(
"create() failed because package node does not exists")
return None
version_criteria = {
'pecosystem': self.ecosystem_package.ecosystem,
'pname': self.ecosystem_package.name,
'version': self.version
}
present_version = Version.find_by_criteria(self.label, pck_obj,
version_criteria)
if present_version is None:
return self.create()
else:
self.id = present_version.id
return self.update()
def populate_from_json(cls, input_json):
# NPM packages with dependencies, versions i.e. Package version
# insertion
logger.info("Instantiating package ...")
package = Package.load_from_json(input_json)
logger.info("Saving package ...")
pkg_id = package.save()
logger.info(" Package node ID: %s" % pkg_id)
if pkg_id is None:
return
version = Version.load_from_json(input_json, package=package)
ver_id = version.save()
logger.info(" Version node ID: %s" % ver_id)
package.create_version_edge(version)
analyses = input_json["analyses"]
if "dependency_snapshot" in analyses:
dependency_snapshot = analyses["dependency_snapshot"]
dependency_pck_list, dependency_ver_list, dependency_type = vdv.load_dependencies(
version.ecosystem_package.ecosystem, dependency_snapshot)
for d_pck, d_ver, d_type in zip(dependency_pck_list,
dependency_ver_list,
dependency_type):
d_pck.save()
if d_pck is None:
continue
d_ver.save()
d_pck.create_version_edge(d_ver)
version.add_edge_dependency(d_ver, d_type)
if "metadata" in analyses:
meta_data = analyses["metadata"]
print(" Adding authors_list")
authors_list = Author.load_from_json(meta_data)
for author in authors_list:
a_id = author.save()
print(" author ID: %s" % a_id)
version.add_edge_author(author)
print(" Adding contributor_list")
contributor_list = Contributor.load_from_json(meta_data)
for contributor in contributor_list:
c_id = contributor.save()
print(" contributor ID: %s" % c_id)
version.add_edge_author(contributor)
# License Information
if "source_licenses" in analyses:
print(" Adding source_licenses")
licenses = set()
license_data = analyses["source_licenses"]
license_details_list, license_counts_list, licenses = LicenseDetails.load_from_json(
license_data)
for used_license, license_count in zip(license_details_list,
license_counts_list):
lic_id = used_license.save()
print(" license_data ID: %s" % lic_id)
version.add_license_edge(used_license, license_count)
version.add_license_attribute(licenses)
# NVD Security Information
if "security_issues" in analyses:
print(" Adding security_issues")
security_data = analyses["security_issues"]
security_list, cvss_score, cve_ids = SecurityDetails.load_from_json(
security_data)
for s, cvss, cve_id in zip(security_list, cvss_score, cve_ids):
ss_id = s.save()
print(" security_data ID: %s" % ss_id)
version.add_security_edge(s, cvss)
version.add_cve_ids(cvss, cve_id)
# GitHub Details
if "github_details" in analyses:
print(" Adding github_details")
github_data = analyses["github_details"]
github_result = GithubResult.load_from_json(github_data)
package.add_github_details_as_attr(github_result)
version.add_edge_github_details(github_result)
# Code Metrics
if "code_metrics" in analyses:
print(" Adding code_metrics")
code_metrics_data = analyses["code_metrics"]
code_metrics = CodeMetricsResult.load_from_json(code_metrics_data)
version.add_code_metrics_edge(code_metrics)
version.add_additional_data_as_attr(code_metrics)
if "blackduck" in analyses:
print("Adding extra security info via blackduck")
blackduck_cve = analyses["blackduck"]
issue_list = bl.load_from_json(blackduck_cve)
for issue in issue_list:
bl_obj = bl.add_blackduck_issue(issue)
version.add_blackduck_cve_edge(bl_obj.id)
def test_create_package_entity():
p = Package()
assert (p.id is None)
assert (p.label == "Package")
assert p.last_updated is None
packages = Package.find_all()
assert (len(packages) == 0)
p = Package.load_from_file('test/data/npm--serve-static-1.7.1.json')
r = p.save()
assert (r is not None)
ls_before = p.last_updated
assert ls_before is not None
assert (Package.count() == 1)
criteria_dict = {'ecosystem': 'npm', 'name': 'serve-static'}
p2 = Package.find_by_criteria('Package', criteria_dict)
assert p2.last_updated == p.last_updated
p.save() # must be an update
assert (Package.count() == 1)
ls_after = p.last_updated
assert ls_after >= ls_before
p.create() # duplicate should not create new node
assert (Package.count() == 1)
Package.delete_by_id(p.id)
assert (Package.count() == 0)
def generate_pkg(
dirpath: str,
diameter: float,
height: float,
pitch: float,
lead_width: float,
author: str,
version: str,
create_date: Optional[str],
) -> None:
# Name according IPC-7351 "Capacitor, Polarized Radial Diameter":
# CAPPRD + Lead Spacing + W Lead Width + D Body Diameter + H Body Height
name = 'CAPPRD{}W{}D{}H{}'.format(format_ipc_dimension(pitch),
format_ipc_dimension(lead_width),
format_ipc_dimension(diameter),
format_ipc_dimension(height))
variant = get_variant(diameter, height, pitch, lead_width)
def _pkg_uuid(identifier: str) -> str:
return uuid('pkg', variant, identifier)
def _create_footprint(footprint_identifier: str, name: str) -> Footprint:
def _fpt_uuid(identifier: str) -> str:
return _pkg_uuid(footprint_identifier + '-' + identifier)
drill = LEAD_WIDTH_TO_DRILL[lead_width]
restring = min(
(0.4 if diameter >= 6.0 else 0.3), # preferred restring
(pitch - drill - 0.25) / 2) # minimum required restring
pad_diameter = drill + (2 * restring) # outer diameter of pad
courtyard_diameter = diameter + (1.0 if diameter >= 10.0 else 0.8)
def _generate_fill_polygon(identifier: str, layer: str) -> Polygon:
polygon = Polygon(
uuid=_fpt_uuid(identifier),
layer=Layer(layer),
width=Width(0.0),
fill=Fill(True),
grab_area=GrabArea(False),
)
if ((pitch - pad_diameter) < 0.6):
# not enough space, use a simplified polygon
vertices = [
(0.0, (diameter / 2) - 0.2, 0.0),
(0.0, (pad_diameter / 2) + 0.2, 0.0),
(pitch / 2, (pad_diameter / 2) + 0.2, -180.0),
(pitch / 2, -(pad_diameter / 2) - 0.2, 0.0),
(0.0, -(pad_diameter / 2) - 0.2, 0.0),
(0.0, -(diameter / 2) + 0.2, 180.0),
(0.0, (diameter / 2) - 0.2, 0.0),
]
else:
vertices = [
(0.0, (diameter / 2) - 0.2, 0.0),
(0.0, 0.0, 0.0),
((pitch / 2) - (pad_diameter / 2) - 0.2, 0.0, -180.0),
((pitch / 2) + (pad_diameter / 2) + 0.2, 0.0, -180.0),
((pitch / 2) - (pad_diameter / 2) - 0.2, 0.0, 0.0),
(0.0, 0.0, 0.0),
(0.0, -(diameter / 2) + 0.2, 180.0),
(0.0, (diameter / 2) - 0.2, 0.0),
]
for vertex in vertices:
polygon.add_vertex(
Vertex(Position(vertex[0], vertex[1]), Angle(vertex[2])))
return polygon
footprint = Footprint(
uuid=_fpt_uuid('footprint'),
name=Name(name),
description=Description(''),
)
footprint.add_pad(
FootprintPad(
uuid=_pkg_uuid('pad-plus'),
side=Side.THT,
shape=Shape.RECT,
position=Position(-pitch / 2, 0),
rotation=Rotation(0),
size=Size(pad_diameter, pad_diameter),
drill=Drill(drill),
))
footprint.add_pad(
FootprintPad(
uuid=_pkg_uuid('pad-minus'),
side=Side.THT,
shape=Shape.ROUND,
position=Position(pitch / 2, 0),
rotation=Rotation(0),
size=Size(pad_diameter, pad_diameter),
drill=Drill(drill),
))
# placement
footprint.add_circle(
Circle(
uuid=_fpt_uuid('circle-placement'),
layer=Layer('top_placement'),
width=Width(0.2),
fill=Fill(False),
grab_area=GrabArea(False),
diameter=Diameter(diameter + 0.2),
position=Position(0.0, 0.0),
))
footprint.add_polygon(
_generate_fill_polygon(
identifier='polygon-placement-fill',
layer='top_placement',
))
# documentation
footprint.add_circle(
Circle(
uuid=_fpt_uuid('circle-documentation'),
layer=Layer('top_documentation'),
width=Width(0.2),
fill=Fill(False),
grab_area=GrabArea(False),
diameter=Diameter(diameter - 0.2),
position=Position(0.0, 0.0),
))
footprint.add_polygon(
_generate_fill_polygon(
identifier='polygon-documentation-fill',
layer='top_documentation',
))
# courtyard
footprint.add_circle(
Circle(
uuid=_fpt_uuid('circle-courtyard'),
layer=Layer('top_courtyard'),
width=Width(0.2),
fill=Fill(False),
grab_area=GrabArea(False),
diameter=Diameter(courtyard_diameter),
position=Position(0.0, 0.0),
))
# texts
footprint.add_text(
StrokeText(
uuid=_fpt_uuid('text-name'),
layer=Layer('top_names'),
height=Height(1.0),
stroke_width=StrokeWidth(0.2),
letter_spacing=LetterSpacing.AUTO,
line_spacing=LineSpacing.AUTO,
align=Align('center bottom'),
position=Position(0.0, (diameter / 2) + 0.8),
rotation=Rotation(0.0),
auto_rotate=AutoRotate(True),
mirror=Mirror(False),
value=Value('{{NAME}}'),
))
footprint.add_text(
StrokeText(
uuid=_fpt_uuid('text-value'),
layer=Layer('top_values'),
height=Height(1.0),
stroke_width=StrokeWidth(0.2),
letter_spacing=LetterSpacing.AUTO,
line_spacing=LineSpacing.AUTO,
align=Align('center top'),
position=Position(0.0, -(diameter / 2) - 0.8),
rotation=Rotation(0.0),
auto_rotate=AutoRotate(True),
mirror=Mirror(False),
value=Value('{{VALUE}}'),
))
return footprint
# package
package = Package(
uuid=_pkg_uuid('pkg'),
name=Name(name),
description=Description(
'Polarized radial electrolytic capacitor.\\n\\n' +
'Diameter: {} mm\\n'.format(diameter) +
'Height: {} mm\\n'.format(height) +
'Lead Spacing: {} mm\\n'.format(pitch) +
'Max. Lead Diameter: {} mm\\n\\n'.format(lead_width) +
'Generated with {}'.format(generator)),
keywords=Keywords(
'electrolytic,capacitor,polarized,radial,c,cap,cpol'),
author=Author(author),
version=Version(version),
created=Created(create_date or now()),
deprecated=Deprecated(False),
category=Category('ee75e31d-f231-41d9-8a3b-bea5114f41e3'),
)
package.add_pad(PackagePad(uuid=_pkg_uuid('pad-plus'), name=Name('+')))
package.add_pad(PackagePad(uuid=_pkg_uuid('pad-minus'), name=Name('-')))
package.add_footprint(
_create_footprint(
footprint_identifier='default',
name='default',
))
# write files
pkg_dir_path = path.join(dirpath, package.uuid)
if not (path.exists(pkg_dir_path) and path.isdir(pkg_dir_path)):
makedirs(pkg_dir_path)
with open(path.join(pkg_dir_path, '.librepcb-pkg'), 'w') as f:
f.write('0.1\n')
with open(path.join(pkg_dir_path, 'package.lp'), 'w') as f:
f.write(str(package))
f.write('\n')
print('Wrote package {}'.format(name))
def create(self):
logger.debug("create() %s - data:\n%s\n" %
(self.label, self.to_json()))
if self.ecosystem_package is None:
logger.error("create() failed because ecosystem_package is None")
return None
package_criteria = {
'ecosystem': self.ecosystem_package.ecosystem,
'name': self.ecosystem_package.name
}
pck_obj = Package.find_by_criteria('Package', package_criteria)
if pck_obj is None:
logger.error(
"create() failed because package node does not exists")
return None
try:
version_criteria = {
'pecosystem': self.ecosystem_package.ecosystem,
'pname': self.ecosystem_package.name,
'version': self.version
}
present_version = Version.find_by_criteria(self.label, pck_obj,
version_criteria)
if present_version is None:
ts = time.time()
query = self.g().addV(self.label). \
property('vertex_label', self.label). \
property('pname', self.ecosystem_package.name). \
property('pecosystem', self.ecosystem_package.ecosystem). \
property('version', self.version). \
property('description', self.description). \
property('dependents_count', self.dependents_count). \
property('shipped_as_downstream', self.shipped_as_downstream). \
property('last_updated', ts)
for pck in self.is_packaged_in:
query.property('is_packaged_in', pck)
for pub in self.is_published_in:
query.property('is_published_in', pub)
results = query.toList()
logger.debug("create() %s - results: %s" %
(self.label, results))
self.last_updated = ts
self.id = results[0].id
logger.debug("results: %s" % (results))
logger.debug("Vertex ID : %s, %s: %s" %
(self.id, self.label, self))
logger.info("---Create--- %s ---NEW = %d" %
(self.label, self.id))
return self.id
else:
logger.debug("Version exists: %s " % present_version.id)
self.last_updated = present_version.last_updated
self.id = present_version.id
logger.info("---Create--- %s ---EXISTS = %d" %
(self.label, self.id))
return self.id
except Exception as e:
logger.error("create() failed: %s" % e)
return None
def test_full_import_and_incr_update():
data_dir = 'test/data'
# Let us make sure that target graph has no metadata
graph_meta = GraphPopulator.get_metadata()
assert (graph_meta is None)
# Full import: insert all the EPVs from the given data source
src_dir = os.path.join(data_dir, 'full_import')
report = import_bulk(data_source=LocalFileSystem(src_dir=src_dir),
book_keeper=None)
assert (report.get('status') == 'Success')
assert (report.get('count_imported_EPVs') == 1)
assert (report.get('last_imported_EPV') == 'npm/serve-static/1.7.1.json')
assert (report.get('max_finished_at') == '2017-02-08T12:26:51.962609')
graph_meta = GraphPopulator.get_metadata()
assert (graph_meta is not None)
assert (graph_meta.last_incr_update_ts == '2017-02-08T12:26:51.962609')
# Incremental update 1:
# Let us mimic a scenario where a new EPV was inserted recently: npm/send/0.10.1
src_dir = os.path.join(data_dir, 'incr_update1')
book_keeping_json = os.path.join(data_dir, 'book_keeping1.json')
report = import_bulk(
data_source=LocalFileSystem(src_dir=src_dir),
book_keeper=JsonBookKeeper(json_file_name=book_keeping_json))
assert (report.get('status') == 'Success')
assert (report.get('count_imported_EPVs') == 1)
assert (report.get('last_imported_EPV') == 'npm/send/0.10.1.json')
assert (report.get('max_finished_at') == '2017-02-22T15:34:59.469864')
graph_meta = GraphPopulator.get_metadata()
assert (graph_meta is not None)
assert (graph_meta.last_incr_update_ts == '2017-02-22T15:34:59.469864')
# Incremental update 2:
# Let us mimic a scenario where a new EPV was inserted recently: npm/parseurl/1.3.1
# and also an already existing EPV was updated recently: npm/serve-static/1.7.1
src_dir = os.path.join(data_dir, 'incr_update2')
book_keeping_json = os.path.join(data_dir, 'book_keeping2.json')
report = import_bulk(
data_source=LocalFileSystem(src_dir=src_dir),
book_keeper=JsonBookKeeper(json_file_name=book_keeping_json))
assert (report.get('status') == 'Success')
assert (report.get('count_imported_EPVs') == 2)
assert (report.get('last_imported_EPV') == 'npm/serve-static/1.7.1.json')
assert (report.get('max_finished_at') == '2017-02-22T15:35:51.962609')
graph_meta = GraphPopulator.get_metadata()
assert (graph_meta is not None)
assert (graph_meta.last_incr_update_ts == '2017-02-22T15:35:51.962609')
# Cleanup
GraphMetaData.delete_all()
assert (GraphMetaData.count() == 0)
LicenseDetails.delete_all()
assert (LicenseDetails.count() == 0)
Author.delete_all()
assert (Author.count() == 0)
CodeMetricsResult.delete_all()
assert (CodeMetricsResult.count() == 0)
CodeMetricsLanguage.delete_all()
assert (CodeMetricsLanguage.count() == 0)
GithubResult.delete_all()
assert (GithubResult.count() == 0)
Contributor.delete_all()
assert (Contributor.count() == 0)
Package.delete_all()
assert (Package.count() == 0)
Version.delete_all()
assert (Version.count() == 0)