def repository_update():
"""Update metadata of a set of repositories."""
# Safeguard
if request.method == 'POST':
request_json = request.get_json()
repo = None
label = None
if request_json is not None:
repo = request_json.get('repolist')
label = request_json.get('repolabel')
# Repository api lib
handler = Repositories()
if repo is not None and label is None:
repo_list = handler.get_repositories(repo)
elif label is not None and repo is None:
repo_list = handler.get_repositories_by_label(label)
else:
repo_list = handler.get_all_repositories()
# Return list of updated repositories
resp = []
if repo_list is not None:
cache.clear()
for repo in repo_list:
handler.select_provider(repo)
ret = handler.update_repository(repo)
if ret is not None and ret > 0:
resp.append(repo.name)
return Response(json.dumps({'repositories': resp}), status=200, mimetype='application/json')
else:
raise InvalidAPIUsage('No repositories configured', status_code=400)
else:
raise InvalidAPIUsage('Invalid request method', status_code=400)
def packages():
result = []
paginator = None
limit = int(request.args.get('limit', 100))
page = int(request.args.get('page', 1))
filter = str(request.args.get('filter', ''))
outdated = str(request.args.get('outdated', ''))
repo = str(request.args.get('repo', ''))
label = str(request.args.get('repolabel', ''))
if outdated == '':
if (10 <= limit <= 100) and page >= 1:
if filter is not None:
filter_string = ('%%%s%%' % filter)
paginator = Package.query.filter(Node.name.like(filter_string)).order_by(Package.name).order_by(
Package.version).paginate(page=page, per_page=limit,
error_out=False)
else:
paginator = Package.query.order_by(Package.name).order_by(Package.version).paginate(page=page,
per_page=limit,
error_out=False)
elif outdated != '' and (repo != '' or label != ''):
# Repository api lib
handler = Repositories()
# Get selected repositories
if repo != '' and label == '':
repo_list = handler.get_repositories(repo)
elif label != '' and repo == '':
repo_list = handler.get_repositories_by_label(label)
else:
raise InvalidAPIUsage('Invalid request', 410)
nodes_filter = ''
if filter != '':
nodes_filter = ('%%%s%%' % filter)
if repo_list:
key = hashlib.sha224((nodes_filter + ','.join(str(repo_list))).encode('utf-8')).hexdigest()
outdated_packages_cache = cache.get(key)
if outdated_packages_cache is not None:
outdated_packages = outdated_packages_cache
else:
outdated_packages = handler.get_outdated_packages(nodes_filter, repo_list)
# Cache for an hour
cache.set(key, outdated_packages, timeout=3600)
length = len(outdated_packages)
paginator = Pagination(page=page, per_page=limit, total_count=length)
paginator.obj_items = outdated_packages
else:
paginator = None
else:
raise InvalidAPIUsage('Invalid request', 410)
if paginator:
for p in paginator.items:
l = len(result)
index = ((l - 1), 0)[0 > (l - 1)]
if l > 0 and result[index]['name'] == p.name:
r = result[index]
ver = {'version': p.version, 'id': p.sha}
r['versions'].append(ver)
else:
r_p = {
'name': p.name,
'versions': [],
'summary': p.summary,
'uri': p.uri,
'provider': p.provider,
'architecture': p.architecture,
'upstream': p.upstream_version
}
ver = {'version': p.version, 'id': p.sha}
r_p['versions'].append(ver)
result.append(r_p)
return Response(json.dumps(result), mimetype='application/json', headers={'Access-Control-Allow-Origin': '*'})
else:
raise InvalidAPIUsage('Invalid API usage', 410)
class TestRepository(TestServerBase):
def test_update_repository(self):
"""Test package list downloading"""
self.repositories = Repositories()
repo = Repository()
repo.id = 1
repo.provider = 'apt'
repo.name = 'Test'
repo.uri = 'http://de.archive.ubuntu.com/ubuntu/dists/precise/main/binary-amd64/Packages.gz'
repo.label = 'Test'
path = os.path.dirname(__file__)
datafile = os.path.join(path, 'data/install_tiny.txt')
data_install = self.process_data(datafile)
test_importer = PackageImport()
test_importer.process_node_info(data_install)
self.repositories.provider = MockRepository()
self.repositories.update_repository(repo)
self.assertEqual(RepoHistory.query.count(), 1)
def test_get_outdated_packages(self):
self.repositories = Repositories()
# create the demo repo
data = {}
data['name'] = 'Repo1'
data['label'] = 'main'
data['uri'] = 'http://de.archive.ubuntu.com/ubuntu/dists/precise/main/binary-amd64/Packages.gz'
data['provider'] = 'apt'
repo_id = self.repositories.create_repository(data)
repo = Repository.query.filter_by(id=repo_id).first()
path = os.path.dirname(__file__)
datafile = os.path.join(path, 'data/install_tiny.txt')
data_install = self.process_data(datafile)
test_importer = PackageImport()
test_importer.process_node_info(data_install)
data = {'75da5ca65e2160e994c33d1006046c2be954efe804a24d34628d0c4cac791ce4': {
"package": "accountsservice",
"filename": "http://us.archive.ubuntu.com/ubuntu/pool/main/a/accountsservice/accountsservice_0.6.21-6ubuntu5.1_amd64.deb",
"description": "query and manipulate user account information",
"version": "0.6.21-6ubuntu5.1",
"architecture": "amd64",
"sha256": "75da5ca65e2160e994c33d1006046c2be954efe804a24d34628d0c4cac791ce4"
},
}
self.repositories.provider = MockRepository(data)
self.repositories.update_repository(repo)
self.assertEqual(RepoHistory.query.count(), 1)
packages = self.repositories.get_outdated_packages('ponyexpress', [repo])
self.assertIsNotNone(packages)
#self.assertNotEqual(packages, [])
self.assertIsInstance(packages[0].upstream_version, dict)
self.assertEqual(len(packages[0].upstream_version), 2)
self.assertEqual(packages[0].pkgversion, '0.6.15-2ubuntu9')
self.assertEqual(packages[0].upstream_version['latest'], '0.6.21-6ubuntu5.1')
def test_get_outdated_packages_multi(self):
self.repositories = Repositories()
# create the demo repo
data = {}
data['name'] = 'Repo1'
data['label'] = 'main'
data['uri'] = 'http://de.archive.ubuntu.com/ubuntu/dists/precise/main/binary-amd64/Packages.gz'
data['provider'] = 'apt'
repo_id = self.repositories.create_repository(data)
repo1 = Repository.query.filter_by(id=repo_id).first()
data = {}
data['name'] = 'Repo2'
data['label'] = 'beta'
data['uri'] = 'http://de.archive.ubuntu.com/ubuntu/dists/trusty/main/binary-amd64/Packages.gz'
data['provider'] = 'apt'
repo_id = self.repositories.create_repository(data)
repo2 = Repository.query.filter_by(id=repo_id).first()
path = os.path.dirname(__file__)
datafile = os.path.join(path, 'data/install_tiny.txt')
data_install = self.process_data(datafile)
test_importer = PackageImport()
test_importer.process_node_info(data_install)
data1 = {'75da5ca65e2160e994c33d1006046c2be954efe804a24d34628d0c4cac791ce4': {
"package": "accountsservice",
"filename": "http://us.archive.ubuntu.com/ubuntu/pool/main/a/accountsservice/accountsservice_0.6.21-6ubuntu5.1_amd64.deb",
"description": "query and manipulate user account information",
"version": "0.6.21-6ubuntu5.1",
"architecture": "amd64",
"sha256": "75da5ca65e2160e994c33d1006046c2be954efe804a24d34628d0c4cac791ce4"
},
}
#self.repositories.select_provider(repo1)
self.repositories.provider = MockRepository(data1)
self.repositories.update_repository(repo1)
data2 = {'2ea5cb960f976935484eff2a2f4f642a991c2af7a88d08b7731dfcd3e3cc3b20': {
"package": "accountsservice",
"filename": "http://us.archive.ubuntu.com/ubuntu/pool/main/a/accountsservice/accountsservice_0.6.35-0ubuntu7_amd64.deb",
"description": "query and manipulate user account information",
"version": "0.6.35-0ubuntu7",
"architecture": "amd64",
"sha256": "2ea5cb960f976935484eff2a2f4f642a991c2af7a88d08b7731dfcd3e3cc3b20"
},
}
#self.repositories.select_provider(repo2)
self.repositories.provider = MockRepository(data2)
self.repositories.update_repository(repo2)
self.assertEqual(RepoHistory.query.count(), 2)
packages = self.repositories.get_outdated_packages('ponyexpress', [repo1, repo2])
self.assertIsNotNone(packages)
self.assertNotEqual(packages, [])
self.assertIsInstance(packages[0].upstream_version, dict)
self.assertGreaterEqual(len(packages[0].upstream_version), 2)
self.assertEqual(packages[0].pkgversion, '0.6.15-2ubuntu9')
self.assertEqual(packages[0].upstream_version['latest'], '0.6.35-0ubuntu7')
def test_version_compare(self):
self.repositories = Repositories()
vers = ['1.0.0', '1.0.1',
'0.5.7', '0.20',
'0.6.15-2ubuntu9', '0.6.15-2ubuntu10',
'0.6.15-2ubuntu9', '0.6.17-2ubuntu9',
'2.2.3.dfsg.1-2build1', '2.2.3.dfsg.1-2build2',
'0.1.1-1ubuntu1'
]
# first two
self.assertEqual(self.repositories.ver_cmp(vers[0], vers[1]), -1)
# third and fourth
self.assertEqual(self.repositories.ver_cmp(vers[2], vers[3]), 1)
# with ubuntu string
self.assertEqual(self.repositories.ver_cmp(vers[4], vers[5]), 1)
#with ubuntu, upstream version change
self.assertEqual(self.repositories.ver_cmp(vers[6], vers[7]), -1)
self.assertEqual(self.repositories.ver_cmp(vers[8], vers[9]), -1)
#
self.assertEqual(self.repositories.ver_cmp(vers[10], vers[10]), 0)
def test_version_parsing(self):
self.repositories = Repositories()
vers = [
"1.024~beta1+svn234",
"1",
"0.6.15-2ubuntu9",
"0.6.15-2ubuntu9.1",
"0.20",
"0.21.1",
"0.1.1-1ubuntu2.1212",
"3.113+nmu3ubuntu3",
"1:2.0.18-1ubuntu1",
"20100513-3.1ubuntu1",
"1.20120910-2",
"2.2.3.dfsg.1-2build1",
"0.3.1~ubuntu4",
"0.9.0-3+wheezy1",
"0.9.7.7ubuntu4",
"1.2.10~pre3-2",
]
# first two
for i in vers:
self.assertEqual(self.repositories.ver_cmp(i, i), 0)
def test_get_repositories(self):
self.repositories = Repositories()
# create the demo repo
data = {}
data['name'] = 'Repo1'
data['label'] = 'main'
data['uri'] = 'http://www.software.repo'
data['provider'] = 'apt'
repo_id = self.repositories.create_repository(data)
expression = '1,2,3'
repo_list = self.repositories.get_repositories(expression)
self.assertIsInstance(repo_list, list)
self.assertEqual(repo_list[0].id, repo_id)
def test_get_repositories_by_label(self):
self.repositories = Repositories()
# create the demo repo
data = {}
data['name'] = 'Repo1'
data['label'] = 'main'
data['uri'] = 'http://www.software.repo'
data['provider'] = 'apt'
repo_id = self.repositories.create_repository(data)
data = {}
data['name'] = 'Repo2'
data['label'] = 'main'
data['uri'] = 'http://www.software.repo'
data['provider'] = 'apt'
repo_id = self.repositories.create_repository(data)
data = {}
data['name'] = 'Repo3'
data['label'] = 'mirror'
data['uri'] = 'http://mirror.repo'
data['provider'] = 'apt'
repo_id = self.repositories.create_repository(data)
repo_list = self.repositories.get_repositories_by_label('main')
self.assertIsInstance(repo_list, list)
self.assertEqual(len(repo_list), 2)
self.assertEqual(repo_list[0].name, 'Repo1')
def test_load_config(self):
yml = u"""
repositories:
- name: Ubuntu
url: http://de.archive.ubuntu.com/ubuntu/dists/precise/main/binary-amd64/Packages.gz
label: ubuntu
provider: apt
- name: Cloud
url: http://ubuntu-cloud.archive.canonical.com/ubuntu/dists/precise-proposed/icehouse/main/binary-amd64/Packages.gz
label: mirror
provider: apt
"""
config = io.StringIO(initial_value=yml)
repoyaml = Repositories.load_config(stream=config)
self.assertIsNotNone(repoyaml)
self.assertIsInstance(repoyaml, list)
self.assertEqual(len(repoyaml), 2)