def analyses(app):
e1 = Ecosystem(name='npm', backend=EcosystemBackend.npm)
p1 = Package(ecosystem=e1, name='arrify')
v1 = Version(package=p1, identifier='1.0.1')
model1 = Analysis(version=v1, started_at=now, finished_at=later)
app.rdb.session.add(model1)
e2 = Ecosystem(name='pypi', backend=EcosystemBackend.pypi)
p2 = Package(ecosystem=e2, name='flexmock')
v2 = Version(package=p2, identifier='0.10.1')
model2 = Analysis(version=v2, started_at=later, access_count=1)
app.rdb.session.add(model2)
app.rdb.session.commit()
worker_results2 = {'a': 'b', 'c': 'd', 'e': 'f', 'g': 'h', 'i': 'j',
'digests': {'details':
[{'artifact': True,
'sha1': '6be7ae55bae2372c7be490321bbe5ead278bb51b'}]}}
for w, tr in worker_results2.items():
app.rdb.session.add(WorkerResult(analysis_id=model2.id, worker=w, task_result=tr))
model3 = Analysis(version=v2, started_at=later, access_count=1,
audit={'audit': {'audit': 'audit', 'e': 'f', 'g': 'h'}, 'a': 'b', 'c': 'd'})
app.rdb.session.add(model3)
app.rdb.session.commit()
worker_results3 = {'digests': {'details':
[{'artifact': True,
'sha1': '6be7ae55bae2372c7be490321bbe5ead278bb51b'}]}}
for w, tr in worker_results3.items():
app.rdb.session.add(WorkerResult(analysis_id=model3.id, worker=w, task_result=tr))
app.rdb.session.commit()
return (model1, model2, model3)
def fill_analyses(app):
ecosystems = [
Ecosystem(name='pypi', backend=EcosystemBackend.pypi, url='https://pypi.python.org/',
fetch_url='https://pypi.python.org/pypi'),
Ecosystem(name='npm', backend=EcosystemBackend.npm, url='https://www.npmjs.com/',
fetch_url='https://registry.npmjs.org/'),
Ecosystem(name='go', backend=EcosystemBackend.scm),
]
packages = [
Package(name='flexmock', ecosystem=ecosystems[0]),
Package(name='requests', ecosystem=ecosystems[0]),
Package(name='sequence', ecosystem=ecosystems[1]),
Package(name='arrify', ecosystem=ecosystems[1]),
Package(name='serve-static', ecosystem=ecosystems[1]),
]
versions = [
Version(identifier='0.10.1', package=packages[0]),
Version(identifier='0.9.1', package=packages[0]),
Version(identifier='2.0.0', package=packages[1]),
Version(identifier='2.2.1', package=packages[2]),
Version(identifier='1.0.1', package=packages[3]),
Version(identifier='1.7.1', package=packages[4]),
]
analyses = [
Analysis(version=versions[0], started_at=now), # pypi/flexmock/0.10.1
Analysis(version=versions[0], started_at=later, access_count=1), # pypi/flexmock/0.10.1
Analysis(version=versions[1], started_at=even_later), # pypi/flexmock/0.9.1
Analysis(version=versions[2], started_at=now), # pypi/requests/2.0.0
Analysis(version=versions[3], started_at=later), # npm/sequence/2.2.1
Analysis(version=versions[4], started_at=now, finished_at=later), # npm/arrify/1.0.1
Analysis(version=versions[5], started_at=now, finished_at=later,
release='npm:serve-static:1.7.1'), # npm/serve-static/1.7.1
]
# worker results that correspond to analyses above
worker_results = [
WorkerResult(worker='digests', analysis=analyses[1],
task_result={'details': [{'artifact': True,
'sha1': '6be7ae55bae2372c7be490321bbe5ead278bb51b'}]}),
WorkerResult(worker='static_analysis', task_result={'details': []}, analysis=analyses[1]),
WorkerResult(worker='source_licenses',
task_result={'schema': {'name': 'source_licenses', 'version': '1-0-0'}},
analysis=analyses[1])
]
package_gh_usage = [
PackageGHUsage(name='arrify', count=100, ecosystem_backend='npm')
]
for a in ecosystems + packages + versions + analyses + worker_results + package_gh_usage:
app.rdb.session.add(a)
app.rdb.session.commit()
return (ecosystems, packages, versions, analyses, worker_results, package_gh_usage)
def test_execute(self, tmpdir):
artifact_digest, artifact_path = IndianaJones.fetch_artifact(
Ecosystem(name='pypi', backend=EcosystemBackend.pypi),
artifact=PYPI_MODULE_NAME,
version=PYPI_MODULE_VERSION,
target_dir=str(tmpdir))
args = dict.fromkeys(('ecosystem', 'name', 'version'), 'some-value')
flexmock(EPVCache).should_receive(
'get_extracted_source_tarball').and_return(str(tmpdir))
flexmock(EPVCache).should_receive('get_source_tarball').and_return(
artifact_path)
task = DigesterTask.create_test_instance(task_name='digests')
results = task.execute(arguments=args)
assert results is not None
assert isinstance(results, dict)
assert set(results.keys()) == {'details', 'status', 'summary'}
artifact_details = None
for details in results['details']:
assert {'sha256', 'sha1', 'md5', 'ssdeep',
'path'}.issubset(set(details.keys()))
if details.get('artifact'):
artifact_details = details
# there are artifact details
assert artifact_details is not None
# the artifact digest which Indy returns is the same as the one from DigesterTask
assert artifact_digest == artifact_details['sha256'] == compute_digest(
artifact_path)
assert artifact_details['path'] == 'six-1.0.0.tar.gz'
def rubygems(rdb):
rubygems = Ecosystem(name='rubygems',
backend=EcosystemBackend.rubygems,
fetch_url='https://rubygems.org/api/v1')
rdb.add(rubygems)
rdb.commit()
return rubygems
def pypi(rdb):
pypi = Ecosystem(name='pypi',
backend=EcosystemBackend.pypi,
fetch_url='https://pypi.python.org/pypi')
rdb.add(pypi)
rdb.commit()
return pypi
def npm(rdb):
npm = Ecosystem(name='npm',
backend=EcosystemBackend.npm,
fetch_url='https://registry.npmjs.org/')
rdb.add(npm)
rdb.commit()
return npm
def maven(rdb):
maven = Ecosystem(name='maven',
backend=EcosystemBackend.maven,
fetch_url='')
rdb.add(maven)
rdb.commit()
return maven
def test_execute(self, tmpdir):
npm = Ecosystem(name='npm', backend=EcosystemBackend.npm)
flexmock(self.m.storage).should_receive('get_ecosystem').with_args(
'npm').and_return(npm)
name = 'wrappy'
version = '1.0.2'
required = {
'homepage', 'version', 'declared_license', 'code_repository',
'bug_reporting', 'description', 'name', 'author'
}
IndianaJones.fetch_artifact(npm,
artifact=name,
version=version,
target_dir=str(tmpdir))
args = {'ecosystem': npm.name, 'name': 'foo', 'version': 'bar'}
flexmock(EPVCache).should_receive(
'get_extracted_source_tarball').and_return(str(tmpdir))
results = self.m.execute(arguments=args)
assert results is not None
assert isinstance(results, dict)
details = results['details'][0]
assert required.issubset(set(
details.keys())) # check at least the required are there
assert all([details[key]
for key in list(required)]) # assert required are not None
assert details['name'] == name
def fill_packages_for_paging(app, request):
e = Ecosystem(name='pypi', backend=EcosystemBackend.pypi)
app.rdb.session.add(e)
for p in range(0, 11):
app.rdb.session.add(Package(ecosystem=e, name=str(p)))
app.rdb.session.commit()
def test_execute_with_mock_anitya(self, ecosystem, project, md5sum,
dist_git):
rdb()
s = create_db_scoped_session()
dummy_homepage = "http://project-homepage.com"
dummy_response = Response()
dummy_response.status_code = 200
s.add(Ecosystem(name='npm', backend=EcosystemBackend.npm))
s.commit()
DownstreamMapCache(
)[md5sum] = dist_git # fill in key-value mapping in cache
task = AnityaTask.create_test_instance(task_name='anitya')
args = {'ecosystem': ecosystem, 'name': project}
flexmock(task).should_receive(
"_get_project_homepage").once().and_return(dummy_homepage)
flexmock(task).should_receive("_get_artifact_hash").once().and_return(
md5sum)
flexmock(task).should_receive(
"_create_anitya_project").once().and_return(dummy_response)
flexmock(task).should_receive(
"_add_downstream_mapping").once().and_return(dummy_response)
results = task.execute(arguments=args)
assert results is None
def setup_method(self, method):
rdb()
self.s = create_db_scoped_session()
self.en = 'foo'
self.pn = 'bar'
self.vi = '1.1.1'
self.e = Ecosystem(name=self.en, backend=EcosystemBackend.maven)
self.p = Package(ecosystem=self.e, name=self.pn)
self.v = Version(package=self.p, identifier=self.vi)
self.a = Analysis(version=self.v, finished_at=datetime.datetime.now())
self.a2 = Analysis(version=self.v,
finished_at=datetime.datetime.now() +
datetime.timedelta(seconds=10))
self.s.add(self.a)
self.s.add(self.a2)
self.s.commit()
self.bp = BayesianPostgres(
connection_string=get_postgres_connection_string())
def _make_ecosystem(name):
return Ecosystem(name=name, backend=getattr(EcosystemBackend, name))
# -*- coding: utf-8 -*-
import pytest
from flexmock import flexmock
from cucoslib.object_cache import EPVCache
from cucoslib.enums import EcosystemBackend
from cucoslib.workers import LinguistTask
from cucoslib.models import Ecosystem
from cucoslib.process import IndianaJones
ECOSYSTEM = Ecosystem(name='pypi', backend=EcosystemBackend.pypi)
MODULE_NAME = 'six'
MODULE_VERSION = '1.10.0'
@pytest.mark.usefixtures("dispatcher_setup")
class TestLinguist(object):
@pytest.mark.usefixtures("no_s3_connection")
def test_execute(self, tmpdir):
IndianaJones.fetch_artifact(
ecosystem=ECOSYSTEM, artifact=MODULE_NAME,
version=MODULE_VERSION, target_dir=str(tmpdir))
args = dict.fromkeys(('ecosystem', 'name', 'version'), 'some-value')
flexmock(EPVCache).should_receive('get_extracted_source_tarball').and_return(str(tmpdir))
task = LinguistTask.create_test_instance(task_name='languages')
results = task.execute(args)
assert results is not None
assert isinstance(results, dict)
assert set(results.keys()) == {'details', 'status', 'summary'}
details = results['details']
# rely on the digest of the npm downloaded tarball matching the upstream one.
# In that case we should probably consider downloading tarballs directly from registry.npmjs.org.
# because for example AnityaTask relies on this.
NPM_MODULE_DIGEST = '8db082250efa45673f344bb809c7cfa1ce37ca9274de29635a40d1e7df6d6114'
PYPI_MODULE_NAME = "six"
PYPI_MODULE_VERSION = "1.0.0"
PYPI_MODULE_DIGEST = 'ca79c14c8cb5e58912d185f0e07ca9c687e232b7c68c4b73bf1c83ef5979333e'
RUBYGEMS_MODULE_NAME = "permutation"
RUBYGEMS_MODULE_VERSION = "0.1.7"
RUBYGEMS_MODULE_DIGEST = 'e715cccaccb8e2d1450fbdda85bbe84963a32e9bf612db278cbb3d6781267638'
MAVEN_MODULE_NAME = "com.rabbitmq:amqp-client"
MAVEN_MODULE_VERSION = "3.6.1"
MAVEN_MODULE_DIGEST = 'cb6cdb7de8d37cb1b15b23867435c7dbbeaa1ca4b766f434138a8b9ef131994f'
npm = Ecosystem(name='npm', backend=EcosystemBackend.npm)
pypi = Ecosystem(name='pypi', backend=EcosystemBackend.pypi)
rubygems = Ecosystem(name='rubygems', backend=EcosystemBackend.rubygems)
maven = Ecosystem(name='maven', backend=EcosystemBackend.maven)
@pytest.fixture
def tmpdir():
tmp = tempfile.mkdtemp()
yield tmp
shutil.rmtree(tmp)
def test_git_add_and_commit_everything_with_dotgit(tmpdir):
# if there's a .git file somewhere in the archive, we don't want it to fail adding
subprocess.check_output(['git', 'init', str(tmpdir)], universal_newlines=True)