def test_version_dependencies():
p = Package.load_from_json(serve_static_json)
assert p.save() is not None
v = Version.load_from_json(serve_static_json, package=p)
v.save()
p.create_version_edge(v)
dependency_data = serve_static_json["analyses"]["dependency_snapshot"]
dependency_pck_list, dependency_ver_list, dependency_type = \
vdv.load_dependencies(v.ecosystem_package.ecosystem, dependency_data)
for d_pck, d_ver, d_type in zip(dependency_pck_list, dependency_ver_list,
dependency_type):
assert d_pck.save() is not None
d_ver.save()
d_pck.create_version_edge(d_ver)
v.add_edge_dependency(d_ver, d_type)
assert (Version.count_dependency(v.id) == 4)
assert (Version.count() + Package.count() == 10)
for pd, vd in zip(dependency_pck_list, dependency_ver_list):
Version.delete_by_id(vd.id)
Package.delete_by_id(pd.id)
Version.delete_by_id(v.id)
Package.delete_by_id(p.id)
def test_version_entity():
p = Package.load_from_json(serve_static_json)
assert p.save() is not None
v = Version.load_from_json(serve_static_json, package=p)
v.save()
assert (Version.count() == 1)
ls_before = v.last_updated
assert ls_before is not None
criteria_dict = {
'pecosystem': 'npm',
'pname': 'serve-static',
'version': '1.7.1'
}
v2 = Version.find_by_criteria('Version', p, criteria_dict)
assert v2.last_updated == v.last_updated
v.save()
ls_after = v.last_updated
assert (Version.count() == 1)
assert ls_after >= ls_before
assert v.last_updated >= v2.last_updated
test_packaged_in = ['nodejs-serve-static']
test_published_in = []
assert (all(pck in test_packaged_in for pck in v.is_packaged_in))
assert (all(pub in test_published_in for pck in v.is_published_in))
# now create an edge
edge_count_before = Package.edge_count()
p.create_version_edge(v)
edge_count_after = Package.edge_count()
assert (edge_count_after == edge_count_before + 1)
# now try to create an edge again
edge_count_before = Package.edge_count()
p.create_version_edge(v)
edge_count_after = Package.edge_count()
assert (edge_count_after == edge_count_before)
# this should return all versions associated with this package
p_versions = p.get_versions()
assert (len(p_versions) == 1)
Version.delete_by_id(v.id)
Package.delete_by_id(p.id)
def test_package_import_from_folder():
package_dir = 'test/data/S3-data/maven/junit:junit'
packages = Package.find_all()
assert (len(packages) == 0)
versions = Version.find_all()
assert (len(versions) == 0)
p = Package()
assert (p.id is None)
assert (p.label == "Package")
assert p.last_updated is None
report = import_from_folder(package_dir)
assert (report.get('status') == 'Success')
assert (report.get('count_imported_EPVs') == 1)
assert (report.get('last_imported_EPV') == '4.8.2.json')
assert (report.get('max_finished_at') == '2017-02-24T13:42:29.665786')
criteria_dict = {'ecosystem': 'maven', 'name': 'junit:junit'}
p1 = Package.find_by_criteria('Package', criteria_dict)
assert p1 is not None
assert p1.id is not None
assert (p1.ecosystem == 'maven')
assert (p1.latest_version == '4.12')
assert (p1.package_dependents_count == -1)
assert (p1.name == 'junit:junit')
criteria_dict = {
'pecosystem': 'maven',
'pname': 'junit:junit',
'version': '4.8.2'
}
v1 = Version.find_by_criteria('Version', p1, criteria_dict)
assert v1 is not None
assert v1.ecosystem_package is not None
assert (v1.ecosystem_package.ecosystem == 'maven')
packages = Package.find_all()
assert (len(packages) == 1)
versions = Version.find_all()
assert (len(versions) == 1)
Package.delete_by_id(p1.id)
assert (Package.count() == 0)
Version.delete_by_id(v1.id)
assert (Version.count() == 0)
GraphMetaData.delete_all()
assert (GraphMetaData.count() == 0)
def test_bucket_import_from_folder():
bucket_dir = 'test/data/S3-data'
packages = Package.find_all()
assert (len(packages) == 0)
p = Package()
assert (p.id is None)
assert (p.label == "Package")
assert p.last_updated is None
report = import_from_folder(bucket_dir)
assert (report.get('status') == 'Success')
assert (report.get('count_imported_EPVs') == 3)
assert (report.get('last_imported_EPV') == 'npm/sequence/3.0.0.json')
assert (report.get('max_finished_at') == '2017-02-24T13:43:11.872916')
criteria_dict = {'ecosystem': 'npm', 'name': 'sequence'}
p1 = Package.find_by_criteria('Package', criteria_dict)
assert p1 is not None
assert p1.id is not None
assert (p1.ecosystem == 'npm')
assert (p1.latest_version == '3.0.0')
assert (p1.package_dependents_count == 22)
assert (p1.name == 'sequence')
criteria_dict = {
'pecosystem': 'npm',
'pname': 'sequence',
'version': '3.0.0'
}
v1 = Version.find_by_criteria('Version', p1, criteria_dict)
assert v1 is not None
assert v1.ecosystem_package is not None
assert (v1.ecosystem_package.ecosystem == 'npm')
packages = Package.find_all()
assert (len(packages) == 3)
versions = Version.find_all()
assert (len(versions) == 4)
Package.delete_all()
assert (Package.count() == 0)
Version.delete_all()
assert (Version.count() == 0)
GraphMetaData.delete_all()
assert (GraphMetaData.count() == 0)
def test_ecosystem_import_from_folder():
ecosystem_dir = 'test/data/S3-data/maven'
packages = Package.find_all()
assert (len(packages) == 0)
p = Package()
assert (p.id is None)
assert (p.label == "Package")
assert p.last_updated is None
report = import_from_folder(ecosystem_dir)
assert (report.get('status') == 'Success')
assert (report.get('count_imported_EPVs') == 2)
assert (
report.get('last_imported_EPV') == 'org.slf4j:slf4j-api/1.5.6.json')
assert (report.get('max_finished_at') == '2017-02-24T13:43:11.872916')
criteria_dict = {'ecosystem': 'maven', 'name': 'junit:junit'}
p1 = Package.find_by_criteria('Package', criteria_dict)
criteria_dict = {
'pecosystem': 'maven',
'pname': 'junit:junit',
'version': '4.8.2'
}
v1 = Version.find_by_criteria('Version', p1, criteria_dict)
assert v1 is not None
assert v1.ecosystem_package is not None
assert (v1.ecosystem_package.ecosystem == 'maven')
packages = Package.find_all()
assert (len(packages) == 2)
versions = Version.find_all()
assert (len(versions) == 3)
Package.delete_all()
assert (Package.count() == 0)
Version.delete_all()
assert (Version.count() == 0)
GraphMetaData.delete_all()
assert (GraphMetaData.count() == 0)
def test_create_package_entity():
packages = Package.find_all()
assert (len(packages) == 0)
list_epv_1 = [{
'ecosystem': 'maven',
'name': 'org.slf4j:slf4j-api',
'version': '1.5.6'
}]
import_epv_from_folder('test/data/S3-data', list_epv=list_epv_1)
criteria_dict = {'ecosystem': 'maven', 'name': 'org.slf4j:slf4j-api'}
p = Package.find_by_criteria('Package', criteria_dict)
assert p.latest_version == '1.7.22'
p.save() # must be an update
assert (Package.count() == 2)
p.create() # duplicate should not create new node
assert (Package.count() == 2)
criteria_dict = {'ecosystem': 'maven', 'name': 'junit:junit'}
p2 = Package.find_by_criteria('Package', criteria_dict)
assert p2.latest_version == '-1'
list_epv_2 = [{
'ecosystem': 'maven',
'name': 'junit:junit',
'version': '4.8.2'
}]
import_epv_from_folder('test/data/S3-data', list_epv=list_epv_2)
criteria_dict = {'ecosystem': 'maven', 'name': 'junit:junit'}
p3 = Package.find_by_criteria('Package', criteria_dict)
assert p3.latest_version == '4.12'
p.save() # must be an update
assert (Package.count() == 2)
Package.delete_all()
assert (Package.count() == 0)
Version.delete_all()
assert (Version.count() == 0)
def test_full_import_and_incr_update():
data_dir = 'test/data'
# Let us make sure that target graph has no metadata
graph_meta = GraphPopulator.get_metadata()
assert (graph_meta is None)
# Full import: insert all the EPVs from the given data source
src_dir = os.path.join(data_dir, 'full_import')
report = import_bulk(data_source=LocalFileSystem(src_dir=src_dir),
book_keeper=None)
assert (report.get('status') == 'Success')
assert (report.get('count_imported_EPVs') == 1)
assert (report.get('last_imported_EPV') == 'npm/serve-static/1.7.1.json')
assert (report.get('max_finished_at') == '2017-02-08T12:26:51.962609')
graph_meta = GraphPopulator.get_metadata()
assert (graph_meta is not None)
assert (graph_meta.last_incr_update_ts == '2017-02-08T12:26:51.962609')
# Incremental update 1:
# Let us mimic a scenario where a new EPV was inserted recently: npm/send/0.10.1
src_dir = os.path.join(data_dir, 'incr_update1')
book_keeping_json = os.path.join(data_dir, 'book_keeping1.json')
report = import_bulk(
data_source=LocalFileSystem(src_dir=src_dir),
book_keeper=JsonBookKeeper(json_file_name=book_keeping_json))
assert (report.get('status') == 'Success')
assert (report.get('count_imported_EPVs') == 1)
assert (report.get('last_imported_EPV') == 'npm/send/0.10.1.json')
assert (report.get('max_finished_at') == '2017-02-22T15:34:59.469864')
graph_meta = GraphPopulator.get_metadata()
assert (graph_meta is not None)
assert (graph_meta.last_incr_update_ts == '2017-02-22T15:34:59.469864')
# Incremental update 2:
# Let us mimic a scenario where a new EPV was inserted recently: npm/parseurl/1.3.1
# and also an already existing EPV was updated recently: npm/serve-static/1.7.1
src_dir = os.path.join(data_dir, 'incr_update2')
book_keeping_json = os.path.join(data_dir, 'book_keeping2.json')
report = import_bulk(
data_source=LocalFileSystem(src_dir=src_dir),
book_keeper=JsonBookKeeper(json_file_name=book_keeping_json))
assert (report.get('status') == 'Success')
assert (report.get('count_imported_EPVs') == 2)
assert (report.get('last_imported_EPV') == 'npm/serve-static/1.7.1.json')
assert (report.get('max_finished_at') == '2017-02-22T15:35:51.962609')
graph_meta = GraphPopulator.get_metadata()
assert (graph_meta is not None)
assert (graph_meta.last_incr_update_ts == '2017-02-22T15:35:51.962609')
# Cleanup
GraphMetaData.delete_all()
assert (GraphMetaData.count() == 0)
LicenseDetails.delete_all()
assert (LicenseDetails.count() == 0)
Author.delete_all()
assert (Author.count() == 0)
CodeMetricsResult.delete_all()
assert (CodeMetricsResult.count() == 0)
CodeMetricsLanguage.delete_all()
assert (CodeMetricsLanguage.count() == 0)
GithubResult.delete_all()
assert (GithubResult.count() == 0)
Contributor.delete_all()
assert (Contributor.count() == 0)
Package.delete_all()
assert (Package.count() == 0)
Version.delete_all()
assert (Version.count() == 0)