Exemplo n.º 1
0
    def testReadmeDownload(self):
        readme_contents = '123'

        # Upload a package with no README.
        contents1 = RootNode(dict(
            foo=FileNode(
                hashes=[self.HASH1]
            )
        ))
        self.put_package('test_user', 'foo', contents1, is_public=True)

        # Upload a package with a README that has the same hash as an existing object.
        # README should now get downloaded.
        contents2 = RootNode(dict(
            README=FileNode(
                hashes=[self.HASH1]
            )
        ))
        self._mock_object('test_user', self.HASH1, readme_contents.encode())
        self.put_package('test_user', 'foo', contents2, is_public=True)
        self.s3_stubber.assert_no_pending_responses()

        # Upload a different package with the same README. Nothing should get downloaded.
        contents3 = RootNode(dict(
            README=FileNode(
                hashes=[self.HASH1]
            ),
            bar=GroupNode(dict())
        ))
        self.put_package('test_user', 'foo2', contents3, is_public=True)
Exemplo n.º 2
0
    def testGetObjects(self):
        hash1 = '1' * 64
        hash2 = '2' * 64
        bad_hash = 'f' * 64

        # hash1 is private; hash2 is both private and public, owned by different users.
        self.put_package(
            'test_user', 'pkg1',
            RootNode(children=dict(foo=FileNode([hash1], dict()))))
        self.put_package(
            'share_with', 'pkg2',
            RootNode(children=dict(foo=FileNode([hash2], dict()))))
        self.put_package(
            'share_with',
            'public_pkg2',
            RootNode(children=dict(foo=FileNode([hash2], dict()))),
            is_public=True)

        def _get_hashes(user, hashes):
            headers = {'Authorization': user} if user else {}
            resp = self.app.post('/api/get_objects',
                                 data=json.dumps(hashes),
                                 content_type='application/json',
                                 headers=headers)
            assert resp.status_code == requests.codes.ok

            data = json.loads(resp.data.decode('utf8'))
            urls = data['urls']
            sizes = data['sizes']
            assert set(urls) == set(sizes)
            assert not set(urls) - set(hashes)
            return set(urls)

        # Anonymous user can only see hash2.
        assert _get_hashes(None, [hash1, hash2]) == {hash2}

        # usr1 can see both.
        assert _get_hashes('test_user', [hash1, hash2]) == {hash1, hash2}

        # usr2 can only see hash2; doesn't matter which user it comes from.
        assert _get_hashes('share_with', [hash1, hash2]) == {hash2}

        # Bogus hashes have no effect.
        assert _get_hashes('share_with', [hash2, bad_hash]) == {hash2}
Exemplo n.º 3
0
    def testFullTextSearch(self):
        user = "******"

        # Packages with READMEs.
        packages = {
            "clinton_email": (
                "On Monday, August 31, the State Department released nearly 7,000 pages of Clinton’s "
                "heavily redacted emails (its biggest release of emails to date)."
            ),
            "wine": (
                "From the UCI Machine Learning Repository: Lichman, M. (2013). UCI Machine Learning Repository. "
                "Irvine, CA: University of California, School of Information and Computer Science."
            ),
            "dogscats": "",
            "nothing": "There are no Clinton's emails or wine data here.",
        }

        for name, readme in packages.items():
            blob_hash = hashlib.sha256(readme.encode()).hexdigest()
            contents = RootNode(dict(
                README=FileNode([blob_hash], dict())
            ))

            self._mock_object(user, blob_hash, readme.encode())
            self.put_package(user, name, contents, is_public=True, tag_latest=True)

        # Package with no README, but more nodes.
        contents2 = RootNode(dict(
            wine=GroupNode(dict()),
            baz=GroupNode(dict())
        ))
        self.put_package(user, "foo", contents2, is_public=True, tag_latest=True)

        # Stemming
        self._test_query("redact", {}, ["test_user/clinton_email"])
        self._test_query("releasing", {}, ["test_user/clinton_email"])

        # Stemming on package name
        self._test_query("no wining", {}, ["test_user/wine", "test_user/foo", "test_user/nothing"])

        # Multiple words
        self._test_query("state department's biggest release", {}, ["test_user/clinton_email"])

        # Keywords in package name
        self._test_query("users dog cat", {}, ["test_user/dogscats"])

        # Order precedence: package name, metadata, README
        self._test_query("clinton", {}, ["test_user/clinton_email", "test_user/nothing"])
        self._test_query("wine", {}, ["test_user/wine", "test_user/foo", "test_user/nothing"])

        # Different keywords match different sources: package name and README.
        self._test_query("nothing wine", {}, ["test_user/nothing"])
Exemplo n.º 4
0
    def testSearchReadmeSnippet(self):
        readme_contents = 'foo' * 1000
        blob_hash = '8db466bdfc3265dd1347843b31ed34af0a0c2e6ff0fd4d6a5853755f0e68b8a0'

        contents = RootNode(dict(
            README=FileNode([blob_hash], dict())
        ))

        self._mock_object(self.user, blob_hash, readme_contents.encode())
        self.put_package(self.user, 'pkg', contents, is_public=True, tag_latest=True)

        resp = self.app.get('/api/search/?q=pkg')
        assert resp.status_code == requests.codes.ok
        data = json.loads(resp.data.decode('utf8'))
        packages = data['packages']

        assert len(packages) == 1
        assert packages[0]['is_public'] is True
        assert packages[0]['is_team'] is False
        assert packages[0]['readme_preview'] == readme_contents[0:1024]
Exemplo n.º 5
0
class PushInstallTestCase(QuiltTestCase):
    """
    Test push and install endpoints.
    """

    HASH1 = 'd146942c9a051553f77d1e00672f2829565c590be972a1330de726a8db223589'
    HASH2 = '4cf37d7f670709346438cf2f2598db630eb34520947308aed55ad5e53f0c1518'
    HASH3 = '46449f44f36ec78364ae846fa47df57870e49d3c6cee59b3682aaf289e6d7586'

    CONTENTS = RootNode(
        dict(foo=TableNode(hashes=[HASH1, HASH2],
                           format=PackageFormat.default.value),
             group1=GroupNode(
                 dict(empty=TableNode(hashes=[],
                                      format=PackageFormat.default.value),
                      group2=GroupNode(
                          dict(bar=TableNode(
                              hashes=[HASH1],
                              format=PackageFormat.default.value))))),
             file=FileNode(hashes=[HASH3], metadata={'q_path': 'example'})))

    CONTENTS_WITH_METADATA = RootNode(
        dict(foo=TableNode(hashes=[HASH1, HASH2],
                           format=PackageFormat.default.value,
                           metadata=dict(important=True)),
             group1=GroupNode(
                 dict(empty=TableNode(hashes=[],
                                      format=PackageFormat.default.value,
                                      metadata=dict(whatever="123")),
                      group2=GroupNode(
                          dict(bar=TableNode([HASH1],
                                             PackageFormat.default.value))))),
             file=FileNode(hashes=[HASH3], metadata=dict())))

    CONTENTS_2 = RootNode(dict(file=FileNode(hashes=[HASH3])))

    HUGE_CONTENTS = RootNode(
        dict(
            README=FileNode(hashes=[HASH1]),
            group1=GroupNode(
                dict(group2=GroupNode(
                    dict(group3=GroupNode(
                        dict(group4=GroupNode(
                            dict(
                                group5=GroupNode(dict(
                                    group6=GroupNode(dict()))))))))))),
            big_group=GroupNode(
                {'child%02d' % i: GroupNode(dict())
                 for i in range(1, 21)})))

    CONTENTS_WITH_Q_EXT = RootNode(
        dict(file1=FileNode(hashes=[HASH1], metadata=dict()),
             file2=FileNode(hashes=[HASH1], metadata=dict(q_path=None)),
             file3=FileNode(hashes=[HASH1], metadata=dict(q_path='')),
             file4=FileNode(hashes=[HASH1],
                            metadata=dict(q_path='a/b.c/d.jpg')),
             file5=FileNode(
                 hashes=[HASH1],
                 metadata=dict(q_path='C:\\Windows\\System32\\clock.exe')),
             file6=FileNode(hashes=[HASH1],
                            metadata=dict(q_path='C:\\foo.bar\\BLAH.JPG')),
             README=FileNode(hashes=[HASH2], metadata=dict(q_path='README'))))

    CONTENTS_HASH = 'a20597100b045f5420de46b7188590e8688bcfe2ac01e9cbefe26f8919b3f44d'
    CONTENTS_2_HASH = 'ede3e3b8d0d3df8503aa9b27d592b5e27281f929cb440a556a2d0c3c52a912e7'

    def testContentsHash(self):
        assert hash_contents(self.CONTENTS) == self.CONTENTS_HASH
        assert hash_contents(self.CONTENTS_WITH_METADATA) == self.CONTENTS_HASH

    @patch('quilt_server.views.ALLOW_ANONYMOUS_ACCESS', True)
    def testSuccessfulPushInstall(self):
        """
        Push a package, then install it.
        """
        # Push a package.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(is_public=True,
                                                 description="",
                                                 contents=self.CONTENTS,
                                                 sizes={
                                                     self.HASH1: 1,
                                                     self.HASH2: 2,
                                                     self.HASH3: 3
                                                 }),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})

        assert resp.status_code == requests.codes.ok

        data = json.loads(resp.data.decode('utf8'))
        assert data[
            'package_url'] == 'http://*****:*****@patch('quilt_server.views.ALLOW_ANONYMOUS_ACCESS', True)
    def testPushNewMetadata(self):
        # Push the original contents.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(is_public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        # Push the metadata.
        resp = self.app.put(
            '/api/package/test_user/foo/%s' % self.CONTENTS_HASH,
            data=json.dumps(dict(is_public=True,
                                 description="",
                                 contents=self.CONTENTS_WITH_METADATA),
                            default=encode_node),
            content_type='application/json',
            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        # Verify that the metadata got saved.
        resp = self.app.get('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        data = json.loads(resp.data.decode('utf8'), object_hook=decode_node)
        assert data['contents'] == self.CONTENTS_WITH_METADATA

    @patch('quilt_server.views.ALLOW_ANONYMOUS_ACCESS', True)
    def testNotLoggedIn(self):
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json')
        assert resp.status_code == requests.codes.unauthorized

        data = json.loads(resp.data.decode('utf8'))
        assert 'message' in data

        resp = self.app.get(
            '/api/package/test_user/foo/%s' % self.CONTENTS_HASH, )
        assert resp.status_code == requests.codes.not_found

        data = json.loads(resp.data.decode('utf8'))
        assert 'message' in data

    def testCreateWrongUser(self):
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'blah'})
        assert resp.status_code == requests.codes.forbidden

        data = json.loads(resp.data.decode('utf8'))
        assert 'message' in data

    def testInvalidRequest(self):
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data='hello',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.bad_request

        data = json.loads(resp.data.decode('utf8'))
        assert 'message' in data

        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict()),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.bad_request

        data = json.loads(resp.data.decode('utf8'))
        assert 'message' in data

    def testInvalidHash(self):
        resp = self.app.put('/api/package/test_user/foo/%s' % self.HASH1,
                            data=json.dumps(dict(description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.bad_request

        data = json.loads(resp.data.decode('utf8'))
        assert 'message' in data

    def testInvalidPackageName(self):
        resp = self.app.put('/api/package/test_user/bad-name/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.bad_request

        data = json.loads(resp.data.decode('utf8'))
        assert 'message' in data

    @patch('quilt_server.views.ALLOW_ANONYMOUS_ACCESS', True)
    def testCase(self):
        # Can't create a package if the username has the wrong case.
        resp = self.app.put('/api/package/Test_User/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(is_public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.forbidden

        # Successfully create a package.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(is_public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        # Can't update with the wrong username case.
        resp = self.app.put('/api/package/Test_User/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(is_public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.forbidden

        # Can't update with the wrong package name case.
        resp = self.app.put('/api/package/test_user/Foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(is_public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.forbidden

        # Can't install with the wrong case.
        # TODO: Special error for this one.
        resp = self.app.get('/api/package/test_user/Foo/%s' %
                            self.CONTENTS_HASH,
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.not_found

    def testGetBlob(self):
        resp = self.app.get('/api/blob/test_user/%s' % self.HASH1,
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok
        data = json.loads(resp.data.decode('utf8'))

        for method in ['head', 'get', 'put']:
            url = urllib.parse.urlparse(data[method])
            assert url.path == '/%s/objs/test_user/%s' % (
                app.config['PACKAGE_BUCKET_NAME'], self.HASH1)

        resp = self.app.get('/api/blob/test_user/%s' % self.HASH1,
                            headers={'Authorization': 'bad_user'})
        assert resp.status_code == requests.codes.forbidden

    @patch('quilt_server.views.ALLOW_ANONYMOUS_ACCESS', True)
    @mock_customer(plan=PaymentPlan.INDIVIDUAL)
    def testCreatePublic(self, customer):
        # Create a new public package.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(is_public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        # Verify that it's public.
        resp = self.app.get('/api/package/test_user/foo/')
        assert resp.status_code == requests.codes.ok

        # Create a private package.
        resp = self.app.put('/api/package/test_user/bar/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})

        # Verify that it's private.
        resp = self.app.get('/api/package/test_user/bar/')
        assert resp.status_code == requests.codes.not_found

        # Try pushing it again as public, and verify that it fails.
        resp = self.app.put('/api/package/test_user/bar/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(is_public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.forbidden

    @mock_customer()
    def testCreatePrivateOnFreePlan(self, customer):
        # Need to upgrade the plan.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={
                                'Authorization': 'test_user',
                            })
        assert resp.status_code == requests.codes.payment_required
        data = json.loads(resp.data.decode('utf8'))
        assert "upgrade your service plan" in data['message']

    def testCreatePrivateNoPayments(self):
        # Payments disabled; no restrictions on private packages.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={
                                'Authorization': 'test_user',
                            })
        assert resp.status_code == requests.codes.ok

    @patch('quilt_server.views.ALLOW_ANONYMOUS_ACCESS', True)
    def testDryRun(self):
        # Create a new package.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(dry_run=True,
                                                 is_public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        # Check that dry run returned signed URLs.
        data = json.loads(resp.data.decode('utf8'))
        urls = data['upload_urls']
        assert len(urls) == 3

        for obj_hash in (self.HASH1, self.HASH2, self.HASH3):
            for method in ('head', 'put'):
                url = urllib.parse.urlparse(urls[obj_hash][method])
                assert url.path == '/%s/objs/test_user/%s' % (
                    app.config['PACKAGE_BUCKET_NAME'], obj_hash)

        # Verify that it doesn't actually exist.
        resp = self.app.get('/api/package/test_user/bar/',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.not_found

        # Check that dry-run errors are useful.
        resp = self.app.put('/api/package/test_user/foo/%s' % 'bad hash',
                            data=json.dumps(dict(dry_run=True,
                                                 is_public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.bad_request

    @patch('quilt_server.views.ALLOW_ANONYMOUS_ACCESS', True)
    def testInstallSubpath(self):
        """
        Push a package, then install it a subpath.
        """
        # Push a package.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(is_public=True,
                                                 description="",
                                                 contents=self.CONTENTS,
                                                 sizes={
                                                     self.HASH1: 1,
                                                     self.HASH2: 2,
                                                     self.HASH3: 3
                                                 }),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        # Install table "foo".
        resp = self.app.get('/api/package/test_user/foo/%s?%s' % (
            self.CONTENTS_HASH,
            urllib.parse.urlencode(dict(subpath='foo')),
        ),
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        data = json.loads(resp.data.decode('utf8'), object_hook=decode_node)
        contents = data['contents']
        assert contents == self.CONTENTS
        assert data['sizes'] == {self.HASH1: 1, self.HASH2: 2}
        urls = data['urls']
        assert len(urls) == 2  # HASH1 and HASH2

        # Install group "group1/group2".
        resp = self.app.get('/api/package/test_user/foo/%s?%s' % (
            self.CONTENTS_HASH,
            urllib.parse.urlencode(dict(subpath='group1/group2')),
        ),
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        data = json.loads(resp.data.decode('utf8'), object_hook=decode_node)
        contents = data['contents']
        assert contents == self.CONTENTS
        urls = data['urls']
        assert len(urls) == 1  # Just HASH1

        # Install a non-existant group.
        resp = self.app.get('/api/package/test_user/foo/%s?%s' % (
            self.CONTENTS_HASH,
            urllib.parse.urlencode(dict(subpath='zzz')),
        ),
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.not_found

        # Install a child of a non-group.
        resp = self.app.get('/api/package/test_user/foo/%s?%s' % (
            self.CONTENTS_HASH,
            urllib.parse.urlencode(dict(subpath='foo/zzz')),
        ),
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.not_found

    @patch('quilt_server.views.ALLOW_ANONYMOUS_ACCESS', True)
    def testPreview(self):
        huge_contents_hash = hash_contents(self.HUGE_CONTENTS)

        readme_contents = 'Hello, World!'
        self._mock_object('test_user', self.HASH1, readme_contents.encode())

        # Push.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            huge_contents_hash,
                            data=json.dumps(dict(is_public=True,
                                                 description="",
                                                 contents=self.HUGE_CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        # Get preview.
        resp = self.app.get('/api/package_preview/test_user/foo/%s' %
                            huge_contents_hash,
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        # Get preview as an anonymous user.
        resp = self.app.get(
            '/api/package_preview/test_user/foo/%s' % huge_contents_hash, )
        assert resp.status_code == requests.codes.ok

        data = json.loads(resp.data.decode('utf8'), object_hook=decode_node)
        assert data['is_public'] == True
        assert data['is_team'] == False
        assert data['readme_url']
        assert data['readme_preview'] == readme_contents
        ts = data['install_timeseries']
        assert ts['startDate'] == ts['endDate']
        assert ts['frequency'] == 'week'
        assert ts['timeSeries'] == [0]
        preview = data['preview']

        assert preview == [
            ['README', None],
            [
                'big_group',
                [
                    ['child01', []],
                    ['child02', []],
                    ['child03', []],
                    ['child04', []],
                    ['child05', []],
                    ['child06', []],
                    ['child07', []],
                    ['child08', []],
                    ['child09', []],
                    ['child10', []],
                    ['...', None],
                ]
            ],
            [
                'group1',
                [['group2', [['group3', [['group4', [['...', None]]]]]]]]
            ],
        ]

        # install as anonymous user
        resp = self.app.get(
            '/api/package/test_user/foo/%s' % huge_contents_hash, )
        assert resp.status_code == requests.codes.ok

        # get new preview
        resp = self.app.get(
            '/api/package_preview/test_user/foo/%s' % huge_contents_hash, )
        assert resp.status_code == requests.codes.ok
        data = json.loads(resp.data.decode('utf8'), object_hook=decode_node)
        ts = data['install_timeseries']
        assert ts['startDate'] == ts['endDate']
        assert ts['frequency'] == 'week'
        assert ts['timeSeries'] == [1]

    @patch('quilt_server.views.ALLOW_ANONYMOUS_ACCESS', True)
    def testPreviewStats(self):
        contents_hash = hash_contents(self.CONTENTS_WITH_Q_EXT)

        readme_contents = 'Hello, World!'
        self._mock_object('test_user', self.HASH2, readme_contents.encode())

        # Push.
        resp = self.app.put('/api/package/test_user/foo/%s' % contents_hash,
                            data=json.dumps(dict(
                                is_public=True,
                                description="",
                                contents=self.CONTENTS_WITH_Q_EXT,
                                sizes={
                                    self.HASH1: 5,
                                    self.HASH2: 37
                                }),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        # Get preview.
        resp = self.app.get('/api/package_preview/test_user/foo/%s' %
                            contents_hash,
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        data = json.loads(resp.data.decode('utf8'), object_hook=decode_node)
        assert data['total_size_uncompressed'] == 42
        assert data['file_types'] == {'': 4, '.jpg': 2, '.exe': 1}
        ts = data['install_timeseries']
        assert ts['startDate'] == ts['endDate']
        assert ts['frequency'] == 'week'
        assert ts['timeSeries'] == [0]

    @patch('quilt_server.views.ALLOW_ANONYMOUS_ACCESS', True)
    def testReadmeDownload(self):
        readme_contents = '123'

        # Upload a package with no README.
        contents1 = RootNode(dict(foo=FileNode(hashes=[self.HASH1])))
        self.put_package('test_user', 'foo', contents1, is_public=True)

        # Upload a package with a README that has the same hash as an existing object.
        # README should now get downloaded.
        contents2 = RootNode(dict(README=FileNode(hashes=[self.HASH1])))
        self._mock_object('test_user', self.HASH1, readme_contents.encode())
        self.put_package('test_user', 'foo', contents2, is_public=True)
        self.s3_stubber.assert_no_pending_responses()

        # Upload a different package with the same README. Nothing should get downloaded.
        contents3 = RootNode(
            dict(README=FileNode(hashes=[self.HASH1]), bar=GroupNode(dict())))
        self.put_package('test_user', 'foo2', contents3, is_public=True)

    @patch('quilt_server.views.ALLOW_ANONYMOUS_ACCESS', True)
    def testInstanceBlob(self):
        # Verify that all blobs are accounted for in the instance<->blob table.

        # Push the first instance with three blobs.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(is_public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        blobs = S3Blob.query.all()
        instance_blobs = db.session.query(InstanceBlobAssoc).all()

        assert len(blobs) == 3
        assert len(instance_blobs) == 3

        # Push the second instance, which reuses one of the blobs.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_2_HASH,
                            data=json.dumps(dict(is_public=True,
                                                 description="",
                                                 contents=self.CONTENTS_2),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        blobs = S3Blob.query.all()
        instance_blobs = db.session.query(InstanceBlobAssoc).all()

        assert len(blobs) == 3
        assert len(instance_blobs) == 4

    @patch('quilt_server.views.ALLOW_ANONYMOUS_ACCESS', True)
    def testTeamAccessFails(self):
        # Verify that --team fails in the public cloud.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(is_team=True,
                                                 description="",
                                                 contents=self.CONTENTS,
                                                 sizes={
                                                     self.HASH1: 1,
                                                     self.HASH2: 2,
                                                     self.HASH3: 3
                                                 }),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})

        assert resp.status_code == requests.codes.forbidden

    @patch('quilt_server.views.ALLOW_ANONYMOUS_ACCESS', False)
    @patch('quilt_server.views.ALLOW_TEAM_ACCESS', True)
    def testTeams(self):
        # Public push fails.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(is_public=True,
                                                 description="",
                                                 contents=self.CONTENTS,
                                                 sizes={
                                                     self.HASH1: 1,
                                                     self.HASH2: 2,
                                                     self.HASH3: 3
                                                 }),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})

        assert resp.status_code == requests.codes.forbidden

        # Team push succeeds.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(is_team=True,
                                                 description="",
                                                 contents=self.CONTENTS,
                                                 sizes={
                                                     self.HASH1: 1,
                                                     self.HASH2: 2,
                                                     self.HASH3: 3
                                                 }),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})

        assert resp.status_code == requests.codes.ok

    @patch('quilt_server.views.ALLOW_ANONYMOUS_ACCESS', True)
    def testOldPublicParamn(self):
        # Push a package using "public" rather than "is_public".
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(public=True,
                                                 description="",
                                                 contents=self.CONTENTS,
                                                 sizes={
                                                     self.HASH1: 1,
                                                     self.HASH2: 2,
                                                     self.HASH3: 3
                                                 }),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})

        assert resp.status_code == requests.codes.ok

        # Verify that "is_public" is set.
        resp = self.app.get('/api/package/test_user/',
                            headers={'Authorization': 'test_user'})

        assert resp.status_code == requests.codes.ok

        data = json.loads(resp.data.decode('utf8'))
        assert data['packages'] == [
            dict(name='foo', is_public=True, is_team=False)
        ]
Exemplo n.º 6
0
class PushInstallTestCase(QuiltTestCase):
    """
    Test push and install endpoints.
    """

    HASH1 = 'd146942c9a051553f77d1e00672f2829565c590be972a1330de726a8db223589'
    HASH2 = '4cf37d7f670709346438cf2f2598db630eb34520947308aed55ad5e53f0c1518'
    HASH3 = '46449f44f36ec78364ae846fa47df57870e49d3c6cee59b3682aaf289e6d7586'

    CONTENTS = RootNode(
        dict(foo=TableNode(hashes=[HASH1, HASH2],
                           format=PackageFormat.default.value),
             group1=GroupNode(
                 dict(empty=TableNode(hashes=[],
                                      format=PackageFormat.default.value),
                      group2=GroupNode(
                          dict(bar=TableNode(
                              hashes=[HASH1],
                              format=PackageFormat.default.value))))),
             file=FileNode(hashes=[HASH3])))

    CONTENTS_WITH_METADATA = RootNode(
        dict(foo=TableNode(hashes=[HASH1, HASH2],
                           format=PackageFormat.default.value,
                           metadata=dict(important=True)),
             group1=GroupNode(
                 dict(empty=TableNode(hashes=[],
                                      format=PackageFormat.default.value,
                                      metadata=dict(whatever="123")),
                      group2=GroupNode(
                          dict(bar=TableNode([HASH1],
                                             PackageFormat.default.value))))),
             file=FileNode(hashes=[HASH3], metadata=dict())))

    HUGE_CONTENTS = RootNode(
        dict(
            README=FileNode(hashes=[HASH1]),
            group1=GroupNode(
                dict(group2=GroupNode(
                    dict(group3=GroupNode(
                        dict(group4=GroupNode(
                            dict(
                                group5=GroupNode(dict(
                                    group6=GroupNode(dict()))))))))))),
            big_group=GroupNode(
                {'child%02d' % i: GroupNode(dict())
                 for i in range(1, 21)})))

    CONTENTS_HASH = 'a20597100b045f5420de46b7188590e8688bcfe2ac01e9cbefe26f8919b3f44d'

    def testContentsHash(self):
        assert hash_contents(self.CONTENTS) == self.CONTENTS_HASH
        assert hash_contents(self.CONTENTS_WITH_METADATA) == self.CONTENTS_HASH

    def testSuccessfulPushInstall(self):
        """
        Push a package, then install it.
        """
        # Push a package.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})

        assert resp.status_code == requests.codes.ok

        # List user's packages.
        resp = self.app.get('/api/package/test_user/',
                            headers={'Authorization': 'test_user'})

        assert resp.status_code == requests.codes.ok

        data = json.loads(resp.data.decode('utf8'))
        assert data['packages'] == [{'name': 'foo', 'is_public': True}]

        # List package instances.
        resp = self.app.get('/api/package/test_user/foo/',
                            headers={'Authorization': 'test_user'})

        assert resp.status_code == requests.codes.ok

        data = json.loads(resp.data.decode('utf8'))
        assert data['hashes'] == [self.CONTENTS_HASH]

        # Install the package.
        resp = self.app.get('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            headers={'Authorization': 'test_user'})

        assert resp.status_code == requests.codes.ok

        data = json.loads(resp.data.decode('utf8'), object_hook=decode_node)
        contents = data['contents']
        assert contents == self.CONTENTS
        assert data['created_by'] == data['updated_by'] == 'test_user'
        assert data['created_at'] == data['updated_at']
        urls = data['urls']
        assert len(urls) == 3

        url1 = urllib.parse.urlparse(urls[self.HASH1])
        url2 = urllib.parse.urlparse(urls[self.HASH2])
        url3 = urllib.parse.urlparse(urls[self.HASH3])
        assert url1.path == '/%s/objs/test_user/%s' % (
            app.config['PACKAGE_BUCKET_NAME'], self.HASH1)
        assert url2.path == '/%s/objs/test_user/%s' % (
            app.config['PACKAGE_BUCKET_NAME'], self.HASH2)
        assert url3.path == '/%s/objs/test_user/%s' % (
            app.config['PACKAGE_BUCKET_NAME'], self.HASH3)

    def testPushNewMetadata(self):
        # Push the original contents.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        # Push the metadata.
        resp = self.app.put(
            '/api/package/test_user/foo/%s' % self.CONTENTS_HASH,
            data=json.dumps(dict(public=True,
                                 description="",
                                 contents=self.CONTENTS_WITH_METADATA),
                            default=encode_node),
            content_type='application/json',
            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        # Verify that the metadata got saved.
        resp = self.app.get('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        data = json.loads(resp.data.decode('utf8'), object_hook=decode_node)
        assert data['contents'] == self.CONTENTS_WITH_METADATA

    def testNotLoggedIn(self):
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json')
        assert resp.status_code == requests.codes.unauthorized

        data = json.loads(resp.data.decode('utf8'))
        assert 'message' in data

        resp = self.app.get(
            '/api/package/test_user/foo/%s' % self.CONTENTS_HASH, )
        assert resp.status_code == requests.codes.not_found

        data = json.loads(resp.data.decode('utf8'))
        assert 'message' in data

    def testCreateWrongUser(self):
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'blah'})
        assert resp.status_code == requests.codes.forbidden

        data = json.loads(resp.data.decode('utf8'))
        assert 'message' in data

    def testInvalidRequest(self):
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data='hello',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.bad_request

        data = json.loads(resp.data.decode('utf8'))
        assert 'message' in data

        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict()),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.bad_request

        data = json.loads(resp.data.decode('utf8'))
        assert 'message' in data

    def testInvalidHash(self):
        resp = self.app.put('/api/package/test_user/foo/%s' % self.HASH1,
                            data=json.dumps(dict(description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.bad_request

        data = json.loads(resp.data.decode('utf8'))
        assert 'message' in data

    def testInvalidPackageName(self):
        resp = self.app.put('/api/package/test_user/bad-name/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.bad_request

        data = json.loads(resp.data.decode('utf8'))
        assert 'message' in data

    def testCase(self):
        # Can't create a package if the username has the wrong case.
        resp = self.app.put('/api/package/Test_User/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.forbidden

        # Successfully create a package.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        # Can't update with the wrong username case.
        resp = self.app.put('/api/package/Test_User/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.forbidden

        # Can't update with the wrong package name case.
        resp = self.app.put('/api/package/test_user/Foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.forbidden

        # Can't install with the wrong case.
        # TODO: Special error for this one.
        resp = self.app.get('/api/package/test_user/Foo/%s' %
                            self.CONTENTS_HASH,
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.not_found

    def testGetBlob(self):
        resp = self.app.get('/api/blob/test_user/%s' % self.HASH1,
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok
        data = json.loads(resp.data.decode('utf8'))

        for method in ['head', 'get', 'put']:
            url = urllib.parse.urlparse(data[method])
            assert url.path == '/%s/objs/test_user/%s' % (
                app.config['PACKAGE_BUCKET_NAME'], self.HASH1)

        resp = self.app.get('/api/blob/test_user/%s' % self.HASH1,
                            headers={'Authorization': 'bad_user'})
        assert resp.status_code == requests.codes.forbidden

    @mock_customer(plan=PaymentPlan.INDIVIDUAL)
    def testCreatePublic(self, customer):
        # Create a new public package.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        # Verify that it's public.
        resp = self.app.get('/api/package/test_user/foo/')
        assert resp.status_code == requests.codes.ok

        # Create a private package.
        resp = self.app.put('/api/package/test_user/bar/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})

        # Verify that it's private.
        resp = self.app.get('/api/package/test_user/bar/')
        assert resp.status_code == requests.codes.not_found

        # Try pushing it again as public, and verify that it fails.
        resp = self.app.put('/api/package/test_user/bar/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.forbidden

    @mock_customer()
    def testCreatePrivateOnFreePlan(self, customer):
        # Need to upgrade the plan.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={
                                'Authorization': 'test_user',
                            })
        assert resp.status_code == requests.codes.payment_required
        data = json.loads(resp.data.decode('utf8'))
        assert "upgrade your service plan" in data['message']

    def testCreatePrivateNoPayments(self):
        # Payments disabled; no restrictions on private packages.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={
                                'Authorization': 'test_user',
                            })
        assert resp.status_code == requests.codes.ok

    def testDryRun(self):
        # Create a new package.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(dry_run=True,
                                                 public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        # Check that dry run returned signed URLs.
        data = json.loads(resp.data.decode('utf8'))
        urls = data['upload_urls']
        assert len(urls) == 3

        for obj_hash in (self.HASH1, self.HASH2, self.HASH3):
            for method in ('head', 'put'):
                url = urllib.parse.urlparse(urls[obj_hash][method])
                assert url.path == '/%s/objs/test_user/%s' % (
                    app.config['PACKAGE_BUCKET_NAME'], obj_hash)

        # Verify that it doesn't actually exist.
        resp = self.app.get('/api/package/test_user/bar/',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.not_found

        # Check that dry-run errors are useful.
        resp = self.app.put('/api/package/test_user/foo/%s' % 'bad hash',
                            data=json.dumps(dict(dry_run=True,
                                                 public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.bad_request

    def testInstallSubpath(self):
        """
        Push a package, then install it a subpath.
        """
        # Push a package.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            self.CONTENTS_HASH,
                            data=json.dumps(dict(public=True,
                                                 description="",
                                                 contents=self.CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        # Install table "foo".
        resp = self.app.get('/api/package/test_user/foo/%s?%s' % (
            self.CONTENTS_HASH,
            urllib.parse.urlencode(dict(subpath='foo')),
        ),
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        data = json.loads(resp.data.decode('utf8'), object_hook=decode_node)
        contents = data['contents']
        assert contents == self.CONTENTS
        urls = data['urls']
        assert len(urls) == 2  # HASH1 and HASH2

        # Install group "group1/group2".
        resp = self.app.get('/api/package/test_user/foo/%s?%s' % (
            self.CONTENTS_HASH,
            urllib.parse.urlencode(dict(subpath='group1/group2')),
        ),
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        data = json.loads(resp.data.decode('utf8'), object_hook=decode_node)
        contents = data['contents']
        assert contents == self.CONTENTS
        urls = data['urls']
        assert len(urls) == 1  # Just HASH1

        # Install a non-existant group.
        resp = self.app.get('/api/package/test_user/foo/%s?%s' % (
            self.CONTENTS_HASH,
            urllib.parse.urlencode(dict(subpath='zzz')),
        ),
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.not_found

        # Install a child of a non-group.
        resp = self.app.get('/api/package/test_user/foo/%s?%s' % (
            self.CONTENTS_HASH,
            urllib.parse.urlencode(dict(subpath='foo/zzz')),
        ),
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.not_found

    def testPreview(self):
        huge_contents_hash = hash_contents(self.HUGE_CONTENTS)

        # Push.
        resp = self.app.put('/api/package/test_user/foo/%s' %
                            huge_contents_hash,
                            data=json.dumps(dict(public=True,
                                                 description="",
                                                 contents=self.HUGE_CONTENTS),
                                            default=encode_node),
                            content_type='application/json',
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        # Get preview.
        resp = self.app.get('/api/package_preview/test_user/foo/%s' %
                            huge_contents_hash,
                            headers={'Authorization': 'test_user'})
        assert resp.status_code == requests.codes.ok

        data = json.loads(resp.data.decode('utf8'), object_hook=decode_node)
        assert data['readme_url']
        preview = data['preview']

        assert preview == [
            ['README', None],
            [
                'big_group',
                [
                    ['child01', []],
                    ['child02', []],
                    ['child03', []],
                    ['child04', []],
                    ['child05', []],
                    ['child06', []],
                    ['child07', []],
                    ['child08', []],
                    ['child09', []],
                    ['child10', []],
                    ['...', None],
                ]
            ],
            [
                'group1',
                [['group2', [['group3', [['group4', [['...', None]]]]]]]]
            ],
        ]
Exemplo n.º 7
0
    def testPushSubpackage(self):
        # Do a normal push.
        resp = self.app.put(
            '/api/package/test_user/foo/%s' % self.CONTENTS_2_HASH,
            data=json.dumps(dict(
                is_public=True,
                description="",
                contents=self.CONTENTS_2,
                sizes={self.HASH3: 3}
            ), default=encode_node),
            content_type='application/json',
            headers={
                'Authorization': 'test_user'
            }
        )
        assert resp.status_code == requests.codes.ok

        # Set the "latest" tag.
        resp = self.app.put(
            '/api/tag/test_user/foo/latest',
            data=json.dumps(dict(
                hash=self.CONTENTS_2_HASH
            )),
            content_type='application/json',
            headers={
                'Authorization': 'test_user'
            }
        )
        assert resp.status_code == requests.codes.ok

        # Now push a subpackage.
        resp = self.app.post(
            '/api/package_update/test_user/foo/group1/group2',
            data=json.dumps(dict(
                is_public=True,
                description="",
                contents=GroupNode(dict()),
                sizes={}
            ), default=encode_node),
            content_type='application/json',
            headers={
                'Authorization': 'test_user'
            }
        )
        assert resp.status_code == requests.codes.ok

        # READMEs get downloaded as usual, too.
        readme_contents = 'Blah'
        self._mock_object('test_user', self.HASH1, readme_contents.encode())
        resp = self.app.post(
            '/api/package_update/test_user/foo/README',
            data=json.dumps(dict(
                is_public=True,
                description="",
                contents=FileNode([self.HASH1]),
                sizes={self.HASH1: 1}
            ), default=encode_node),
            content_type='application/json',
            headers={
                'Authorization': 'test_user'
            }
        )
        assert resp.status_code == requests.codes.ok

        package_hash = json.loads(resp.data.decode('utf-8'))['package_hash']

        # Install the package and verify that it has everything.
        resp = self.app.get(
            '/api/tag/test_user/foo/latest',
            headers={
                'Authorization': 'test_user'
            }
        )
        assert resp.status_code == requests.codes.ok
        data = json.loads(resp.data.decode('utf8'))

        # Verify that the "latest" tag points to the last hash we got from a subpackage push.
        assert data['hash'] == package_hash

        resp = self.app.get(
            '/api/package/test_user/foo/%s' % package_hash,
            headers={
                'Authorization': 'test_user'
            }
        )
        assert resp.status_code == requests.codes.ok
        data = json.loads(resp.data.decode('utf8'), object_hook=decode_node)
        contents = data['contents']

        # Verify that the contents is everything we've pushed so far.
        assert contents == RootNode(dict(
            file=FileNode([self.HASH3]),
            README=FileNode([self.HASH1]),
            group1=GroupNode(dict(
                group2=GroupNode(dict())
            ))
        ))
Exemplo n.º 8
0
    def testFullTextSearch(self):
        # Packages with READMEs.
        packages = {
            "clinton_email":
            ("On Monday, August 31, the State Department released nearly 7,000 pages of Clinton’s "
             "heavily redacted emails (its biggest release of emails to date)."
             ),
            "wine":
            ("From the UCI Machine Learning Repository: Lichman, M. (2013). UCI Machine Learning Repository. "
             "Irvine, CA: University of California, School of Information and Computer Science."
             ),
            "dogscats":
            "",
            "nothing":
            "There are no Clinton's emails or wine data here.",
        }

        for name, readme in packages.items():
            blob_hash = hashlib.sha256(readme.encode()).hexdigest()
            contents = RootNode(dict(README=FileNode([blob_hash], dict())))

            self._mock_object(self.user, blob_hash, readme.encode())
            self.put_package(self.user,
                             name,
                             contents,
                             is_public=True,
                             tag_latest=True)

        # Package with no README, but more nodes.
        contents2 = RootNode(
            dict(wine=GroupNode(dict()), baz=GroupNode(dict())))
        self.put_package(self.user,
                         "foo",
                         contents2,
                         is_public=True,
                         tag_latest=True)

        def _test_query(query, headers, expected_results):
            params = dict(q=query)
            resp = self.app.get('/api/search/?%s' %
                                urllib.parse.urlencode(params),
                                headers=headers)

            assert resp.status_code == requests.codes.ok
            data = json.loads(resp.data.decode('utf8'))

            results = ['%(owner)s/%(name)s' % pkg for pkg in data['packages']]
            assert results == expected_results

        # Stemming
        _test_query("redact", {}, ["test_user/clinton_email"])
        _test_query("releasing", {}, ["test_user/clinton_email"])

        # Stemming on package name
        _test_query("no wining", {},
                    ["test_user/wine", "test_user/foo", "test_user/nothing"])

        # Multiple words
        _test_query("state department's biggest release", {},
                    ["test_user/clinton_email"])

        # Keywords in package name
        _test_query("users dog cat", {}, ["test_user/dogscats"])

        # Order precedence: package name, metadata, README
        _test_query("clinton", {},
                    ["test_user/clinton_email", "test_user/nothing"])
        _test_query("wine", {},
                    ["test_user/wine", "test_user/foo", "test_user/nothing"])

        # Different keywords match different sources: package name and README.
        _test_query("nothing wine", {}, ["test_user/nothing"])