def test_deserializable_collection(self):
fj = FileBSON(self.directory, self.name, [])
content1 = DeserializableClass("1", "2")
content2 = DeserializableClass("3", "4")
content = serializable_elist([content1, content2])
fj.write(content)
content_r = fj.read()
self.assertEqual(content, content_r)
def test_deserializable_collection(self):
fj = FileJSON(self.directory, self.name, [])
content1 = DeserializableClass("1", "2")
content2 = DeserializableClass("3", "4")
content = serializable_elist([content1, content2])
fj.write(content)
content_r = fj.read()
self.assertEqual(content, content_r)
def get_ebuild_data(pp, common_config={}, config={}):
ebuild_data = {}
package, version = pp['name'], pp['version']
if 'description' in pp:
description = pp['description']
else:
description = ''
ebuild_data["realname"] = package
ebuild_data["realversion"] = version
ebuild_data["description"] = description
ebuild_data["longdescription"] = description
ebuild_data["homepage"] = pp["repository"]["url"]
pkg_license = ''
if "license" in pp:
pkg_license = pp["license"]
pkg_license = convert(
[common_config, config], "licenses", pkg_license)
ebuild_data["license"] = pkg_license
ebuild_data["source_uri"] = pp["dist"]["tarball"]
ebuild_data["md5"] = pp["dist"]["shasum"]
if 'dependencies' in pp:
ll = serializable_elist(separator="\n\t")
for dep in pp["dependencies"].items():
# print(dep)
ll.append(Dependency('dev-nodejs', dep[0]))
#print(ll, '@@@@@')
ebuild_data["dependencies"] = ll
else:
ebuild_data["dependencies"] = ''
if 'devDependencies' in pp:
ebuild_data["devDependencies"] = pp["devDependencies"]
else:
ebuild_data["devDependencies"] = ''
return ebuild_data
def process_data(self, pkg_db, data, common_config, config):
"""
Process downloaded and parsed data and generate tree.
Args:
pkg_db: Package database.
data: Dictionary with data, keys are file names.
common_config; Backend config.
config: Repository config.
"""
archive_contents = data['archive-contents']
repo_uri = config["repo_uri"]
if sexpdata.car(archive_contents) != 1:
raise SyncError('sync failed: ' \
+ repo_uri + ' bad archive contents format')
category = 'app-emacs'
pkg_db.add_category(category)
common_data = {'eclasses' : ['g-sorcery', 'gs-elpa'],
'maintainer' : [{'email' : '*****@*****.**',
'name' : 'Jauhien Piatlicki'}],
'homepage' : repo_uri,
'repo_uri' : repo_uri
}
pkg_db.set_common_data(category, common_data)
PKG_INFO = 2
PKG_NAME = 0
INFO_VERSION = 0
INFO_DEPENDENCIES = 1
INFO_DESCRIPTION = 2
INFO_SRC_TYPE = 3
DEP_NAME = 0
#DEP_VERSION = 1 #we do not use it at the moment
for entry in sexpdata.cdr(archive_contents):
desc = entry[PKG_INFO].value()
realname = entry[PKG_NAME].value()
if self.in_config([common_config, config], "exclude", realname):
continue
pkg = Package("app-emacs", realname,
'.'.join(map(str, desc[INFO_VERSION])))
source_type = desc[INFO_SRC_TYPE].value()
allowed_ords = set(range(ord('a'), ord('z'))) \
| set(range(ord('A'), ord('Z'))) | \
set(range(ord('0'), ord('9'))) | set(list(map(ord,
['+', '_', '-', ' ', '.', '(', ')', '[', ']', '{', '}', ','])))
description = "".join([x for x in desc[INFO_DESCRIPTION] if ord(x) in allowed_ords])
deps = desc[INFO_DEPENDENCIES]
#fix for crappy arhive-contents that have "No commentary."
#in place of dependency
if isinstance(deps, basestring):
deps = []
dependencies = serializable_elist(separator="\n\t")
for dep in deps:
dep = self.convert_dependency([common_config, config],
dep[DEP_NAME].value(), external = False)
if dep:
dependencies.append(dep)
properties = {'source_type' : source_type,
'description' : description,
'dependencies' : dependencies,
'depend' : dependencies,
'rdepend' : dependencies,
'realname' : realname,
'longdescription' : description
}
pkg_db.add_package(pkg, properties)
def test_functionality(self):
port = 8080
for fmt in SUPPORTED_FILE_FORMATS:
sync_address = "127.0.0.1:" + str(port) + "/dummy.tar.gz"
orig_tempdir = TemporaryDirectory()
orig_path = os.path.join(orig_tempdir.name, "db")
os.makedirs(orig_path)
orig_db = PackageDB(orig_path, preferred_category_format=fmt)
orig_db.add_category("app-test1")
orig_db.add_category("app-test2")
ebuild_data = {
"test1":
"tst1",
"test2":
"tst2",
"test3":
serializable_elist([
DeserializableClass("1", "2"),
DeserializableClass("3", "4")
])
}
common_data = {
"common1":
"cmn1",
"common2":
"cmn2",
"common3":
serializable_elist([
DeserializableClass("c1", "c2"),
DeserializableClass("c3", "c4")
])
}
packages = [
Package("app-test1", "test", "1"),
Package("app-test1", "test", "2"),
Package("app-test1", "test1", "1"),
Package("app-test2", "test2", "1")
]
for package in packages:
orig_db.add_package(package, ebuild_data)
orig_db.set_common_data("app-test1", common_data)
full_data = dict(ebuild_data)
full_data.update(common_data)
orig_db.write()
os.system("cd " + orig_tempdir.name +
" && tar cvzf good.tar.gz db")
os.system("echo invalid >> " + orig_tempdir.name +
"/db/app-test1/packages." + fmt)
os.system("cd " + orig_tempdir.name +
" && tar cvzf dummy.tar.gz db")
test_db = PackageDB(self.tempdir.name)
self.assertRaises(SyncError, test_db.sync, sync_address)
srv = Server(orig_tempdir.name, port=port)
srv.start()
try:
self.assertRaises(IntegrityError, test_db.sync, sync_address)
os.system("cd " + orig_tempdir.name +
" && mv good.tar.gz dummy.tar.gz")
test_db.sync(sync_address)
finally:
srv.shutdown()
srv.join()
test_db.read()
self.assertEqual(orig_db.database, test_db.database)
self.assertEqual(orig_db.get_common_data("app-test1"),
test_db.get_common_data("app-test1"))
self.assertEqual(orig_db.get_common_data("app-test2"),
test_db.get_common_data("app-test2"))
self.assertEqual(set(test_db.list_categories()),
set(["app-test1", "app-test2"]))
self.assertTrue(test_db.in_category("app-test1", "test"))
self.assertFalse(test_db.in_category("app-test2", "test"))
self.assertRaises(InvalidKeyError, test_db.in_category,
"app-test3", "test")
self.assertEqual(set(test_db.list_package_names("app-test1")),
set(['test', 'test1']))
self.assertEqual(
set(test_db.list_catpkg_names()),
set(['app-test1/test', 'app-test1/test1', 'app-test2/test2']))
self.assertRaises(InvalidKeyError, test_db.list_package_versions,
"invalid", "test")
self.assertRaises(InvalidKeyError, test_db.list_package_versions,
"app-test1", "invalid")
self.assertEqual(
set(test_db.list_package_versions("app-test1", "test")),
set(['1', '2']))
self.assertEqual(set(test_db.list_all_packages()), set(packages))
self.assertEqual(test_db.get_package_description(packages[0]),
full_data)
self.assertRaises(KeyError, test_db.get_package_description,
Package("invalid", "invalid", "1"))
self.assertEqual(test_db.get_max_version("app-test1", "test"), "2")
self.assertEqual(test_db.get_max_version("app-test1", "test1"),
"1")
self.assertRaises(InvalidKeyError, test_db.get_max_version,
"invalid", "invalid")
pkg_set = set(packages)
for package, data in test_db:
self.assertTrue(package in pkg_set)
if package.category == "app-test1":
self.assertEqual(data, full_data)
else:
self.assertEqual(data, ebuild_data)
pkg_set.remove(package)
self.assertTrue(not pkg_set)
self.assertEqual(orig_db.database, test_db.database)
port = port + 1
def process_data(self, pkg_db, data, common_config, config):
"""
Process parsed package data.
"""
category = "dev-python"
pkg_db.add_category(category)
common_data = {}
common_data["eclasses"] = ['g-sorcery', 'gs-pypi']
common_data["maintainer"] = [{'email' : '*****@*****.**',
'name' : 'Jauhien Piatlicki'}]
common_data["dependencies"] = serializable_elist(separator="\n\t")
pkg_db.set_common_data(category, common_data)
#todo: write filter functions
allowed_ords_pkg = set(range(ord('a'), ord('z') + 1)) | set(range(ord('A'), ord('Z') + 1)) | \
set(range(ord('0'), ord('9') + 1)) | set(list(map(ord,
['+', '_', '-'])))
allowed_ords_desc = set(range(ord('a'), ord('z') + 1)) | set(range(ord('A'), ord('Z') + 1)) | \
set(range(ord('0'), ord('9') + 1)) | set(list(map(ord,
['+', '_', '-', ' ', '.', '(', ')', '[', ']', '{', '}', ','])))
now = datetime.datetime.now()
pseudoversion = "%04d%02d%02d" % (now.year, now.month, now.day)
for (package, version), description in data["packages"]["index"].items():
pkg = package + "-" + version
if not pkg in data["packages"]:
continue
pkg_data = data["packages"][pkg]
if not pkg_data["files"] and not pkg_data["info"]:
continue
files_src_uri = ""
md5 = ""
if pkg_data["files"]:
for file_entry in pkg_data["files"]:
if file_entry["type"] == "\n Source\n ":
files_src_uri = file_entry["url"]
md5 = file_entry["md5"]
break
download_url = ""
info = pkg_data["info"]
if info:
if "Download URL:" in info:
download_url = info["Download URL:"]
if download_url:
source_uri = download_url #todo: find how to define src_uri
else:
source_uri = files_src_uri
if not source_uri:
continue
homepage = ""
pkg_license = ""
py_versions = []
if info:
if "Home Page:" in info:
homepage = info["Home Page:"]
categories = {}
if "Categories" in info:
categories = info["Categories"]
if 'Programming Language' in categories:
for entry in categories['Programming Language']:
if entry == '2':
py_versions.extend(['2_7'])
elif entry == '3':
py_versions.extend(['3_3', '3_4', '3_5'])
elif entry == '2.6':
py_versions.extend(['2_7'])
elif entry == '2.7':
py_versions.extend(['2_7'])
elif entry == '3.2':
py_versions.extend(['3_3'])
elif entry == '3.3':
py_versions.extend(['3_3'])
elif entry == '3.4':
py_versions.extend(['3_4'])
elif entry == '3.5':
py_versions.extend(['3_5'])
if "License" in categories:
pkg_license = categories["License"][-1]
pkg_license = self.convert([common_config, config], "licenses", pkg_license)
if not py_versions:
py_versions = ['2_7', '3_3', '3_4', '3_5']
if len(py_versions) == 1:
python_compat = '( python' + py_versions[0] + ' )'
else:
python_compat = '( python{' + py_versions[0]
for ver in py_versions[1:]:
python_compat += ',' + ver
python_compat += '} )'
filtered_package = "".join([x for x in package if ord(x) in allowed_ords_pkg])
description = "".join([x for x in description if ord(x) in allowed_ords_desc])
filtered_version = version
match_object = re.match("(^[0-9]+[a-z]?$)|(^[0-9][0-9\.]+[0-9][a-z]?$)",
filtered_version)
if not match_object:
filtered_version = pseudoversion
ebuild_data = {}
ebuild_data["realname"] = package
ebuild_data["realversion"] = version
ebuild_data["description"] = description
ebuild_data["longdescription"] = description
ebuild_data["homepage"] = homepage
ebuild_data["license"] = pkg_license
ebuild_data["source_uri"] = source_uri
ebuild_data["md5"] = md5
ebuild_data["python_compat"] = python_compat
pkg_db.add_package(Package(category, filtered_package, filtered_version), ebuild_data)
def test_functionality(self):
port = 8080
for fmt in SUPPORTED_FILE_FORMATS:
sync_address = "127.0.0.1:" + str(port) + "/dummy.tar.gz"
orig_tempdir = TemporaryDirectory()
orig_path = os.path.join(orig_tempdir.name, "db")
os.makedirs(orig_path)
orig_db = PackageDB(orig_path, preferred_category_format=fmt)
orig_db.add_category("app-test1")
orig_db.add_category("app-test2")
ebuild_data = {"test1": "tst1", "test2": "tst2",
"test3": serializable_elist([DeserializableClass("1", "2"),
DeserializableClass("3", "4")])}
common_data = {"common1": "cmn1", "common2": "cmn2",
"common3": serializable_elist([DeserializableClass("c1", "c2"),
DeserializableClass("c3", "c4")])}
packages = [Package("app-test1", "test", "1"), Package("app-test1", "test", "2"),
Package("app-test1", "test1", "1"), Package("app-test2", "test2", "1")]
for package in packages:
orig_db.add_package(package, ebuild_data)
orig_db.set_common_data("app-test1", common_data)
full_data = dict(ebuild_data)
full_data.update(common_data)
orig_db.write()
os.system("cd " + orig_tempdir.name + " && tar cvzf good.tar.gz db")
os.system("echo invalid >> " + orig_tempdir.name + "/db/app-test1/packages." + fmt)
os.system("cd " + orig_tempdir.name + " && tar cvzf dummy.tar.gz db")
test_db = PackageDB(self.tempdir.name)
self.assertRaises(SyncError, test_db.sync, sync_address)
srv = Server(orig_tempdir.name, port=port)
srv.start()
self.assertRaises(IntegrityError, test_db.sync, sync_address)
os.system("cd " + orig_tempdir.name + " && mv good.tar.gz dummy.tar.gz")
test_db.sync(sync_address)
srv.shutdown()
srv.join()
test_db.read()
self.assertEqual(orig_db.database, test_db.database)
self.assertEqual(orig_db.get_common_data("app-test1"), test_db.get_common_data("app-test1"))
self.assertEqual(orig_db.get_common_data("app-test2"), test_db.get_common_data("app-test2"))
self.assertEqual(set(test_db.list_categories()), set(["app-test1", "app-test2"]))
self.assertTrue(test_db.in_category("app-test1", "test"))
self.assertFalse(test_db.in_category("app-test2", "test"))
self.assertRaises(InvalidKeyError, test_db.in_category, "app-test3", "test")
self.assertEqual(set(test_db.list_package_names("app-test1")), set(['test', 'test1']))
self.assertEqual(set(test_db.list_catpkg_names()),set(['app-test1/test', 'app-test1/test1', 'app-test2/test2']))
self.assertRaises(InvalidKeyError, test_db.list_package_versions, "invalid", "test")
self.assertRaises(InvalidKeyError, test_db.list_package_versions, "app-test1", "invalid")
self.assertEqual(set(test_db.list_package_versions("app-test1", "test")), set(['1', '2']))
self.assertEqual(set(test_db.list_all_packages()), set(packages))
self.assertEqual(test_db.get_package_description(packages[0]), full_data)
self.assertRaises(KeyError, test_db.get_package_description, Package("invalid", "invalid", "1"))
self.assertEqual(test_db.get_max_version("app-test1", "test"), "2")
self.assertEqual(test_db.get_max_version("app-test1", "test1"), "1")
self.assertRaises(InvalidKeyError, test_db.get_max_version, "invalid", "invalid")
pkg_set = set(packages)
for package, data in test_db:
self.assertTrue(package in pkg_set)
if package.category == "app-test1":
self.assertEqual(data, full_data)
else:
self.assertEqual(data, ebuild_data)
pkg_set.remove(package)
self.assertTrue(not pkg_set)
self.assertEqual(orig_db.database, test_db.database)
port = port + 1
def process_data(self, pkg_db, data, common_config, config):
"""
Process parsed package data.
"""
category = "dev-python"
pkg_db.add_category(category)
common_data = {}
common_data["eclasses"] = ['g-sorcery', 'gs-pypi']
common_data["maintainer"] = [{
'email': '*****@*****.**',
'name': 'Jauhien Piatlicki'
}]
common_data["dependencies"] = serializable_elist(separator="\n\t")
pkg_db.set_common_data(category, common_data)
#todo: write filter functions
allowed_ords_pkg = set(range(ord('a'), ord('z') + 1)) | set(range(ord('A'), ord('Z') + 1)) | \
set(range(ord('0'), ord('9') + 1)) | set(list(map(ord,
['+', '_', '-'])))
allowed_ords_desc = set(range(ord('a'), ord('z') + 1)) | set(range(ord('A'), ord('Z') + 1)) | \
set(range(ord('0'), ord('9') + 1)) | set(list(map(ord,
['+', '_', '-', ' ', '.', '(', ')', '[', ']', '{', '}', ','])))
now = datetime.datetime.now()
pseudoversion = "%04d%02d%02d" % (now.year, now.month, now.day)
for (package,
version), description in data["packages"]["index"].items():
pkg = package + "-" + version
if not pkg in data["packages"]:
continue
pkg_data = data["packages"][pkg]
if not pkg_data["files"] and not pkg_data["info"]:
continue
files_src_uri = ""
md5 = ""
if pkg_data["files"]:
for file_entry in pkg_data["files"]:
if file_entry["type"] == "\n Source\n ":
files_src_uri = file_entry["url"]
md5 = file_entry["md5"]
break
download_url = ""
info = pkg_data["info"]
if info:
if "Download URL:" in info:
download_url = info["Download URL:"]
if download_url:
source_uri = download_url #todo: find how to define src_uri
else:
source_uri = files_src_uri
if not source_uri:
continue
homepage = ""
pkg_license = ""
py_versions = []
if info:
if "Home Page:" in info:
homepage = info["Home Page:"]
categories = {}
if "Categories" in info:
categories = info["Categories"]
if 'Programming Language' in categories:
for entry in categories['Programming Language']:
if entry == '2':
py_versions.extend(['2_7'])
elif entry == '3':
py_versions.extend(['3_3', '3_4', '3_5'])
elif entry == '2.6':
py_versions.extend(['2_7'])
elif entry == '2.7':
py_versions.extend(['2_7'])
elif entry == '3.2':
py_versions.extend(['3_3'])
elif entry == '3.3':
py_versions.extend(['3_3'])
elif entry == '3.4':
py_versions.extend(['3_4'])
elif entry == '3.5':
py_versions.extend(['3_5'])
if "License" in categories:
pkg_license = categories["License"][-1]
pkg_license = self.convert([common_config, config], "licenses",
pkg_license)
if not py_versions:
py_versions = ['2_7', '3_3', '3_4', '3_5']
if len(py_versions) == 1:
python_compat = '( python' + py_versions[0] + ' )'
else:
python_compat = '( python{' + py_versions[0]
for ver in py_versions[1:]:
python_compat += ',' + ver
python_compat += '} )'
filtered_package = "".join(
[x for x in package if ord(x) in allowed_ords_pkg])
description = "".join(
[x for x in description if ord(x) in allowed_ords_desc])
filtered_version = version
match_object = re.match(
"(^[0-9]+[a-z]?$)|(^[0-9][0-9\.]+[0-9][a-z]?$)",
filtered_version)
if not match_object:
filtered_version = pseudoversion
ebuild_data = {}
ebuild_data["realname"] = package
ebuild_data["realversion"] = version
ebuild_data["description"] = description
ebuild_data["longdescription"] = description
ebuild_data["homepage"] = homepage
ebuild_data["license"] = pkg_license
ebuild_data["source_uri"] = source_uri
ebuild_data["md5"] = md5
ebuild_data["python_compat"] = python_compat
pkg_db.add_package(
Package(category, filtered_package, filtered_version),
ebuild_data)
