Esempio n. 1
0
    def test_version_cache(self):
        library_key = ndb.Key(Library, 'a/b')
        Version(id='v2.0.0', sha='x', status=Status.ready,
                parent=library_key).put()
        Version(id='v1.0.0', sha='x', status=Status.ready,
                parent=library_key).put()
        Version(id='v3.0.0', sha='x', status=Status.ready,
                parent=library_key).put()
        Version(id='v3.0.X', sha='x', status=Status.ready,
                parent=library_key).put()
        Version(id='v4.0.0', sha='x', status=Status.error,
                parent=library_key).put()
        Version(id='v5.0.0',
                sha='x',
                status=Status.pending,
                parent=library_key).put()
        Version(id='xxx', sha='x', status=Status.ready,
                parent=library_key).put()
        versions = yield Library.versions_for_key_async(library_key)
        self.assertEqual(versions, [])

        latest_changed = VersionCache.update(library_key)
        self.assertTrue(latest_changed)
        versions = yield Library.versions_for_key_async(library_key)
        self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0'])

        Version(id='v6.0.0', sha='x', status=Status.ready,
                parent=library_key).put()
        latest_changed = VersionCache.update(library_key)
        self.assertTrue(latest_changed)
        versions = yield Library.versions_for_key_async(library_key)
        self.assertEqual(versions,
                         ['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0', 'v6.0.0'])
Esempio n. 2
0
  def test_analyzer_index_empty(self):
    metadata = """{
      "full_name": "full-name"
    }"""
    library_key = Library(id='owner/repo', metadata=metadata).put()
    version_key = Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put()

    content = Content(id='analysis', parent=version_key, status=Status.pending)
    data = {"analyzerData": {}}
    content.json = data
    content.status = Status.ready
    content.put()

    VersionCache.update(library_key)

    response = self.app.get(util.update_indexes_task('owner', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
    self.assertEqual(response.status_int, 200)

    index = search.Index('repo')
    document = index.get('owner/repo')
    self.assertIsNotNone(document)
    self.assertTrue(len(document.fields) > 0)

    elements = [field for field in document.fields if field.name == 'element']
    self.assertEqual(len(elements), 0)

    behaviors = [field for field in document.fields if field.name == 'behavior']
    self.assertEqual(len(behaviors), 0)
Esempio n. 3
0
    def test_update_doesnt_ingest_older_versions(self):
        library_key = Library(id='org/repo',
                              tags=['v0.1.0', 'v1.0.0', 'v2.0.0'],
                              spdx_identifier='MIT').put()
        Version(id='v1.0.0',
                parent=library_key,
                sha="old",
                status=Status.ready).put()
        VersionCache.update(library_key)

        self.respond_to_github('https://api.github.com/repos/org/repo',
                               {'status': 304})
        self.respond_to_github(
            'https://api.github.com/repos/org/repo/contributors',
            {'status': 304})
        self.respond_to_github(
            'https://api.github.com/repos/org/repo/tags', """[
        {"name": "v0.5.0", "commit": {"sha": "new"}},
        {"name": "v1.0.0", "commit": {"sha": "old"}}
    ]""")
        self.respond_to_github(
            'https://api.github.com/repos/org/repo/stats/participation', '{}')

        response = self.app.get(util.update_library_task('org/repo'),
                                headers={'X-AppEngine-QueueName': 'default'})
        self.assertEqual(response.status_int, 200)

        tasks = self.tasks.get_filtered_tasks()
        self.assertEqual([], [task.url for task in tasks])
Esempio n. 4
0
  def test_update_indexes(self):
    metadata = """{
      "full_name": "full-name"
    }"""
    collection_library_key = Library(id='my/collection', status=Status.ready, kind='collection', metadata=metadata).put()
    collection_version_key = Version(id='v1.0.0', parent=collection_library_key, sha='sha', status=Status.ready).put()
    Content(id='bower', parent=collection_version_key, content="""{"dependencies": {
      "a": "org/element-1#1.0.0",
      "b": "org/element-2#1.0.0"
    }}""").put()
    VersionCache.update(collection_library_key)

    response = self.app.get(util.update_indexes_task('my', 'collection'), headers={'X-AppEngine-QueueName': 'default'})
    self.assertEqual(response.status_int, 200)

    # Triggers ingestions
    tasks = self.tasks.get_filtered_tasks()
    self.assertEqual([
        util.ensure_library_task('org', 'element-1'),
        util.ensure_library_task('org', 'element-2'),
    ], [task.url for task in tasks])

    # Ensures collection references
    ref1 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-1"))
    self.assertIsNotNone(ref1)

    ref2 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-2"))
    self.assertIsNotNone(ref2)
Esempio n. 5
0
  def test_update_indexes(self):
    metadata = """{
      "full_name": "full-name"
    }"""
    collection_library_key = Library(id='my/collection', status=Status.ready, kind='collection', metadata=metadata).put()
    collection_version_key = Version(id='v1.0.0', parent=collection_library_key, sha='sha', status=Status.ready).put()
    Content(id='bower', parent=collection_version_key, content="""{"dependencies": {
      "a": "org/element-1#1.0.0",
      "b": "org/element-2#1.0.0"
    }}""").put()
    VersionCache.update(collection_library_key)

    response = self.app.get(util.update_indexes_task('my', 'collection'), headers={'X-AppEngine-QueueName': 'default'})
    self.assertEqual(response.status_int, 200)

    # Triggers ingestions
    tasks = self.tasks.get_filtered_tasks()
    self.assertEqual([
        util.ensure_library_task('org', 'element-1'),
        util.ensure_library_task('org', 'element-2'),
    ], [task.url for task in tasks])

    # Ensures collection references
    ref1 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-1"))
    self.assertIsNotNone(ref1)

    ref2 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-2"))
    self.assertIsNotNone(ref2)

    # Validate search index
    index = search.Index('repo')
    document = index.get('my/collection')
    self.assertIsNotNone(document)
    self.assertTrue(len(document.fields) > 0)
Esempio n. 6
0
  def test_delete_version(self):
    library_key = ndb.Key(Library, 'owner/repo')
    version_key = Version(id='v1.0.0', parent=library_key, sha='1', status=Status.ready).put()
    VersionCache.update(library_key)

    response = self.app.get('/task/delete/owner/repo/v1.0.0', headers={'X-AppEngine-QueueName': 'default'})
    self.assertEqual(response.status_int, 200)
    version = version_key.get()
    self.assertIsNone(version)
    self.assertEqual(Library.versions_for_key_async(library_key).get_result(), [])
Esempio n. 7
0
  def test_delete_version(self):
    library_key = ndb.Key(Library, 'owner/repo')
    version_key = Version(id='v1.0.0', parent=library_key, sha='1', status=Status.ready).put()
    VersionCache.update(library_key)

    response = self.app.get('/task/delete/owner/repo/v1.0.0', headers={'X-AppEngine-QueueName': 'default'})
    self.assertEqual(response.status_int, 200)
    version = version_key.get()
    self.assertIsNone(version)
    self.assertEqual(Library.versions_for_key_async(library_key).get_result(), [])
Esempio n. 8
0
 def handle_get(self, owner, repo, version):
   # FIXME: Make deletion transactional with check on library that tag is excluded.
   version_key = ndb.Key(Library, Library.id(owner, repo), Version, version)
   ndb.delete_multi(ndb.Query(ancestor=version_key).iter(keys_only=True))
   if VersionCache.update(version_key.parent()):
     task_url = util.update_indexes_task(owner, repo)
     util.new_task(task_url, target='manage')
Esempio n. 9
0
 def handle_get(self, scope, package, version):
   # FIXME: Make deletion transactional with check on library that tag is excluded.
   version_key = ndb.Key(Library, Library.id(scope, package), Version, version)
   ndb.delete_multi(ndb.Query(ancestor=version_key).iter(keys_only=True))
   if VersionCache.update(version_key.parent()):
     task_url = util.update_indexes_task(scope, package)
     util.new_task(task_url, target='manage')
Esempio n. 10
0
  def test_analyze_latest(self):
    library_key = Library(id='owner/repo').put()
    Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put()
    version_key = Version(id='v1.1.2', parent=library_key, sha='sha', status='ready').put()
    VersionCache.update(library_key)

    response = self.app.get('/task/analyze/owner/repo/True', headers={'X-AppEngine-QueueName': 'default'})
    self.assertEqual(response.status_int, 200)

    content = Content.get_by_id('analysis', parent=version_key)
    self.assertEqual(content.get_json(), None)
    self.assertEqual(content.status, Status.pending)

    tasks = self.tasks.get_filtered_tasks()
    self.assertEqual([
        util.ingest_analysis_task('owner', 'repo', 'v1.1.2'),
    ], [task.url for task in tasks])
Esempio n. 11
0
  def test_subsequent_update_triggers_version_deletion(self):
    library_key = Library(id='org/repo', spdx_identifier='MIT', tag_map='{"v1.0.0":"old","v2.0.0":"old"}').put()
    Version(id='v0.1.0', parent=library_key, sha="old", status=Status.ready).put()
    Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put()
    Version(id='v2.0.0', parent=library_key, sha="old", status=Status.ready).put()
    VersionCache.update(library_key)

    self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
    self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
    self.respond_to_github('https://api.github.com/repos/org/repo/tags', {'status': 304})
    self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')

    response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
    self.assertEqual(response.status_int, 200)

    tasks = self.tasks.get_filtered_tasks()
    self.assertEqual([
        util.delete_task('org', 'repo', 'v0.1.0'),
    ], [task.url for task in tasks])
Esempio n. 12
0
  def test_subsequent_update_triggers_version_deletion(self):
    library_key = Library(id='org/repo', spdx_identifier='MIT', tag_map='{"v1.0.0":"old","v2.0.0":"old"}').put()
    Version(id='v0.1.0', parent=library_key, sha="old", status=Status.ready).put()
    Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put()
    Version(id='v2.0.0', parent=library_key, sha="old", status=Status.ready).put()
    VersionCache.update(library_key)

    self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
    self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
    self.respond_to_github('https://api.github.com/repos/org/repo/tags', {'status': 304})
    self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')

    response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
    self.assertEqual(response.status_int, 200)

    tasks = self.tasks.get_filtered_tasks()
    self.assertEqual([
        util.delete_task('org', 'repo', 'v0.1.0'),
    ], [task.url for task in tasks])
Esempio n. 13
0
  def test_update_doesnt_ingest_older_versions(self):
    library_key = Library(id='org/repo', tags=['v0.1.0', 'v1.0.0', 'v2.0.0'], spdx_identifier='MIT').put()
    Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put()
    VersionCache.update(library_key)

    self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
    self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
    self.respond_to_github('https://api.github.com/repos/org/repo/tags', """[
        {"name": "v0.5.0", "commit": {"sha": "new"}},
        {"name": "v1.0.0", "commit": {"sha": "old"}}
    ]""")
    self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')

    response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
    self.assertEqual(response.status_int, 200)

    tasks = self.tasks.get_filtered_tasks()
    self.assertEqual([
    ], [task.url for task in tasks])
Esempio n. 14
0
  def test_version_cache(self):
    library_key = ndb.Key(Library, 'a/b')
    Version(id='v2.0.0', sha='x', status=Status.ready, parent=library_key).put()
    Version(id='v1.0.0', sha='x', status=Status.ready, parent=library_key).put()
    Version(id='v3.0.0', sha='x', status=Status.ready, parent=library_key).put()
    Version(id='v3.0.X', sha='x', status=Status.ready, parent=library_key).put()
    Version(id='v4.0.0', sha='x', status=Status.error, parent=library_key).put()
    Version(id='v5.0.0', sha='x', status=Status.pending, parent=library_key).put()
    Version(id='xxx', sha='x', status=Status.ready, parent=library_key).put()
    versions = yield Library.versions_for_key_async(library_key)
    self.assertEqual(versions, [])

    latest_changed = VersionCache.update(library_key)
    self.assertTrue(latest_changed)
    versions = yield Library.versions_for_key_async(library_key)
    self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0'])

    Version(id='v6.0.0', sha='x', status=Status.ready, parent=library_key).put()
    latest_changed = VersionCache.update(library_key)
    self.assertTrue(latest_changed)
    versions = yield Library.versions_for_key_async(library_key)
    self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0', 'v6.0.0'])
Esempio n. 15
0
 def update_versions_and_index(self):
   if VersionCache.update(self.version_key.parent()):
     task_url = util.update_indexes_task(self.owner, self.repo)
     util.new_task(task_url, target='manage', transactional=True)
Esempio n. 16
0
 def update_versions_and_index(self):
     if VersionCache.update(self.version_key.parent()):
         task_url = util.update_indexes_task(self.owner, self.repo)
         util.new_task(task_url, target='manage', transactional=True)