Exemple #1
0
def repository_update():
    """Update metadata of a set of repositories."""

    # Safeguard
    if request.method == 'POST':
        request_json = request.get_json()

        repo = None
        label = None

        if request_json is not None:
            repo = request_json.get('repolist')
            label = request_json.get('repolabel')

        # Repository api lib
        handler = Repositories()

        if repo is not None and label is None:
            repo_list = handler.get_repositories(repo)
        elif label is not None and repo is None:
            repo_list = handler.get_repositories_by_label(label)
        else:
            repo_list = handler.get_all_repositories()

        # Return list of updated repositories
        resp = []

        if repo_list is not None:

            cache.clear()

            for repo in repo_list:
                handler.select_provider(repo)
                ret = handler.update_repository(repo)

                if ret is not None and ret > 0:
                    resp.append(repo.name)

            return Response(json.dumps({'repositories': resp}), status=200, mimetype='application/json')
        else:
            raise InvalidAPIUsage('No repositories configured', status_code=400)

    else:
        raise InvalidAPIUsage('Invalid request method', status_code=400)
Exemple #2
0
def packages():
    result = []

    paginator = None

    limit = int(request.args.get('limit', 100))
    page = int(request.args.get('page', 1))

    filter = str(request.args.get('filter', ''))

    outdated = str(request.args.get('outdated', ''))
    repo = str(request.args.get('repo', ''))
    label = str(request.args.get('repolabel', ''))

    if outdated == '':
        if (10 <= limit <= 100) and page >= 1:

            if filter is not None:
                filter_string = ('%%%s%%' % filter)
                paginator = Package.query.filter(Node.name.like(filter_string)).order_by(Package.name).order_by(
                    Package.version).paginate(page=page, per_page=limit,
                                              error_out=False)
            else:
                paginator = Package.query.order_by(Package.name).order_by(Package.version).paginate(page=page,
                                                                                                    per_page=limit,
                                                                                                    error_out=False)
    elif outdated != '' and (repo != '' or label != ''):
        # Repository api lib
        handler = Repositories()

        # Get selected repositories
        if repo != '' and label == '':
            repo_list = handler.get_repositories(repo)
        elif label != '' and repo == '':
            repo_list = handler.get_repositories_by_label(label)
        else:
            raise InvalidAPIUsage('Invalid request', 410)

        nodes_filter = ''
        if filter != '':
            nodes_filter = ('%%%s%%' % filter)

        if repo_list:
            key = hashlib.sha224((nodes_filter + ','.join(str(repo_list))).encode('utf-8')).hexdigest()

            outdated_packages_cache = cache.get(key)

            if outdated_packages_cache is not None:
                outdated_packages = outdated_packages_cache
            else:
                outdated_packages = handler.get_outdated_packages(nodes_filter, repo_list)

                # Cache for an hour
                cache.set(key, outdated_packages, timeout=3600)

            length = len(outdated_packages)
            paginator = Pagination(page=page, per_page=limit, total_count=length)
            paginator.obj_items = outdated_packages
        else:
            paginator = None
    else:
        raise InvalidAPIUsage('Invalid request', 410)

    if paginator:
        for p in paginator.items:

            l = len(result)
            index = ((l - 1), 0)[0 > (l - 1)]

            if l > 0 and result[index]['name'] == p.name:
                r = result[index]
                ver = {'version': p.version, 'id': p.sha}
                r['versions'].append(ver)
            else:
                r_p = {
                    'name': p.name,
                    'versions': [],
                    'summary': p.summary,
                    'uri': p.uri,
                    'provider': p.provider,
                    'architecture': p.architecture,
                    'upstream': p.upstream_version
                }
                ver = {'version': p.version, 'id': p.sha}
                r_p['versions'].append(ver)

                result.append(r_p)

        return Response(json.dumps(result), mimetype='application/json', headers={'Access-Control-Allow-Origin': '*'})
    else:
        raise InvalidAPIUsage('Invalid API usage', 410)
class TestRepository(TestServerBase):

    def test_update_repository(self):
        """Test package list downloading"""

        self.repositories = Repositories()

        repo = Repository()
        repo.id = 1
        repo.provider = 'apt'
        repo.name = 'Test'
        repo.uri = 'http://de.archive.ubuntu.com/ubuntu/dists/precise/main/binary-amd64/Packages.gz'
        repo.label = 'Test'

        path = os.path.dirname(__file__)
        datafile = os.path.join(path, 'data/install_tiny.txt')
        data_install = self.process_data(datafile)

        test_importer = PackageImport()
        test_importer.process_node_info(data_install)

        self.repositories.provider = MockRepository()

        self.repositories.update_repository(repo)

        self.assertEqual(RepoHistory.query.count(), 1)

    def test_get_outdated_packages(self):
        self.repositories = Repositories()

        # create the demo repo
        data = {}
        data['name'] = 'Repo1'
        data['label'] = 'main'
        data['uri'] = 'http://de.archive.ubuntu.com/ubuntu/dists/precise/main/binary-amd64/Packages.gz'
        data['provider'] = 'apt'

        repo_id = self.repositories.create_repository(data)
        repo = Repository.query.filter_by(id=repo_id).first()

        path = os.path.dirname(__file__)
        datafile = os.path.join(path, 'data/install_tiny.txt')
        data_install = self.process_data(datafile)

        test_importer = PackageImport()
        test_importer.process_node_info(data_install)

        data = {'75da5ca65e2160e994c33d1006046c2be954efe804a24d34628d0c4cac791ce4': {
                "package": "accountsservice",
                "filename": "http://us.archive.ubuntu.com/ubuntu/pool/main/a/accountsservice/accountsservice_0.6.21-6ubuntu5.1_amd64.deb",
                "description": "query and manipulate user account information",
                "version": "0.6.21-6ubuntu5.1",
                "architecture": "amd64",
                "sha256": "75da5ca65e2160e994c33d1006046c2be954efe804a24d34628d0c4cac791ce4"
            },
        }

        self.repositories.provider = MockRepository(data)

        self.repositories.update_repository(repo)

        self.assertEqual(RepoHistory.query.count(), 1)

        packages = self.repositories.get_outdated_packages('ponyexpress', [repo])

        self.assertIsNotNone(packages)
        #self.assertNotEqual(packages, [])

        self.assertIsInstance(packages[0].upstream_version, dict)
        self.assertEqual(len(packages[0].upstream_version), 2)

        self.assertEqual(packages[0].pkgversion, '0.6.15-2ubuntu9')
        self.assertEqual(packages[0].upstream_version['latest'], '0.6.21-6ubuntu5.1')

    def test_get_outdated_packages_multi(self):
        self.repositories = Repositories()

        # create the demo repo
        data = {}
        data['name'] = 'Repo1'
        data['label'] = 'main'
        data['uri'] = 'http://de.archive.ubuntu.com/ubuntu/dists/precise/main/binary-amd64/Packages.gz'
        data['provider'] = 'apt'

        repo_id = self.repositories.create_repository(data)
        repo1 = Repository.query.filter_by(id=repo_id).first()

        data = {}
        data['name'] = 'Repo2'
        data['label'] = 'beta'
        data['uri'] = 'http://de.archive.ubuntu.com/ubuntu/dists/trusty/main/binary-amd64/Packages.gz'
        data['provider'] = 'apt'

        repo_id = self.repositories.create_repository(data)
        repo2 = Repository.query.filter_by(id=repo_id).first()

        path = os.path.dirname(__file__)
        datafile = os.path.join(path, 'data/install_tiny.txt')
        data_install = self.process_data(datafile)

        test_importer = PackageImport()
        test_importer.process_node_info(data_install)

        data1 = {'75da5ca65e2160e994c33d1006046c2be954efe804a24d34628d0c4cac791ce4': {
                "package": "accountsservice",
                "filename": "http://us.archive.ubuntu.com/ubuntu/pool/main/a/accountsservice/accountsservice_0.6.21-6ubuntu5.1_amd64.deb",
                "description": "query and manipulate user account information",
                "version": "0.6.21-6ubuntu5.1",
                "architecture": "amd64",
                "sha256": "75da5ca65e2160e994c33d1006046c2be954efe804a24d34628d0c4cac791ce4"
            },
        }

        #self.repositories.select_provider(repo1)
        self.repositories.provider = MockRepository(data1)

        self.repositories.update_repository(repo1)

        data2 = {'2ea5cb960f976935484eff2a2f4f642a991c2af7a88d08b7731dfcd3e3cc3b20': {
                "package": "accountsservice",
                "filename": "http://us.archive.ubuntu.com/ubuntu/pool/main/a/accountsservice/accountsservice_0.6.35-0ubuntu7_amd64.deb",
                "description": "query and manipulate user account information",
                "version": "0.6.35-0ubuntu7",
                "architecture": "amd64",
                "sha256": "2ea5cb960f976935484eff2a2f4f642a991c2af7a88d08b7731dfcd3e3cc3b20"
            },
        }

        #self.repositories.select_provider(repo2)
        self.repositories.provider = MockRepository(data2)
        self.repositories.update_repository(repo2)

        self.assertEqual(RepoHistory.query.count(), 2)

        packages = self.repositories.get_outdated_packages('ponyexpress', [repo1, repo2])

        self.assertIsNotNone(packages)
        self.assertNotEqual(packages, [])

        self.assertIsInstance(packages[0].upstream_version, dict)
        self.assertGreaterEqual(len(packages[0].upstream_version), 2)

        self.assertEqual(packages[0].pkgversion, '0.6.15-2ubuntu9')
        self.assertEqual(packages[0].upstream_version['latest'], '0.6.35-0ubuntu7')

    def test_version_compare(self):
        self.repositories = Repositories()

        vers = ['1.0.0', '1.0.1',
                '0.5.7', '0.20',
                '0.6.15-2ubuntu9', '0.6.15-2ubuntu10',
                '0.6.15-2ubuntu9', '0.6.17-2ubuntu9',
                '2.2.3.dfsg.1-2build1', '2.2.3.dfsg.1-2build2',
                '0.1.1-1ubuntu1'
        ]

        # first two
        self.assertEqual(self.repositories.ver_cmp(vers[0], vers[1]), -1)

        # third and fourth
        self.assertEqual(self.repositories.ver_cmp(vers[2], vers[3]), 1)

        # with ubuntu string
        self.assertEqual(self.repositories.ver_cmp(vers[4], vers[5]), 1)

        #with ubuntu, upstream version change
        self.assertEqual(self.repositories.ver_cmp(vers[6], vers[7]), -1)

        self.assertEqual(self.repositories.ver_cmp(vers[8], vers[9]), -1)
        #
        self.assertEqual(self.repositories.ver_cmp(vers[10], vers[10]), 0)

    def test_version_parsing(self):
        self.repositories = Repositories()

        vers = [
            "1.024~beta1+svn234",
            "1",
            "0.6.15-2ubuntu9",
            "0.6.15-2ubuntu9.1",
            "0.20",
            "0.21.1",
            "0.1.1-1ubuntu2.1212",
            "3.113+nmu3ubuntu3",
            "1:2.0.18-1ubuntu1",
            "20100513-3.1ubuntu1",
            "1.20120910-2",
            "2.2.3.dfsg.1-2build1",
            "0.3.1~ubuntu4",
            "0.9.0-3+wheezy1",
            "0.9.7.7ubuntu4",
            "1.2.10~pre3-2",
            ]

        # first two
        for i in vers:
            self.assertEqual(self.repositories.ver_cmp(i, i), 0)

    def test_get_repositories(self):
        self.repositories = Repositories()

        # create the demo repo
        data = {}
        data['name'] = 'Repo1'
        data['label'] = 'main'
        data['uri'] = 'http://www.software.repo'
        data['provider'] = 'apt'
        repo_id = self.repositories.create_repository(data)

        expression = '1,2,3'

        repo_list = self.repositories.get_repositories(expression)
        self.assertIsInstance(repo_list, list)
        self.assertEqual(repo_list[0].id, repo_id)

    def test_get_repositories_by_label(self):
        self.repositories = Repositories()

        # create the demo repo
        data = {}
        data['name'] = 'Repo1'
        data['label'] = 'main'
        data['uri'] = 'http://www.software.repo'
        data['provider'] = 'apt'
        repo_id = self.repositories.create_repository(data)

        data = {}
        data['name'] = 'Repo2'
        data['label'] = 'main'
        data['uri'] = 'http://www.software.repo'
        data['provider'] = 'apt'
        repo_id = self.repositories.create_repository(data)

        data = {}
        data['name'] = 'Repo3'
        data['label'] = 'mirror'
        data['uri'] = 'http://mirror.repo'
        data['provider'] = 'apt'
        repo_id = self.repositories.create_repository(data)

        repo_list = self.repositories.get_repositories_by_label('main')

        self.assertIsInstance(repo_list, list)
        self.assertEqual(len(repo_list), 2)
        self.assertEqual(repo_list[0].name, 'Repo1')

    def test_load_config(self):
        yml = u"""
        repositories:
          - name: Ubuntu
            url: http://de.archive.ubuntu.com/ubuntu/dists/precise/main/binary-amd64/Packages.gz
            label: ubuntu
            provider: apt
          - name: Cloud
            url: http://ubuntu-cloud.archive.canonical.com/ubuntu/dists/precise-proposed/icehouse/main/binary-amd64/Packages.gz
            label: mirror
            provider: apt
        """

        config = io.StringIO(initial_value=yml)

        repoyaml = Repositories.load_config(stream=config)

        self.assertIsNotNone(repoyaml)
        self.assertIsInstance(repoyaml, list)

        self.assertEqual(len(repoyaml), 2)