def test_deserializable_collection(self): fj = FileBSON(self.directory, self.name, []) content1 = DeserializableClass("1", "2") content2 = DeserializableClass("3", "4") content = serializable_elist([content1, content2]) fj.write(content) content_r = fj.read() self.assertEqual(content, content_r)
def test_deserializable_collection(self): fj = FileJSON(self.directory, self.name, []) content1 = DeserializableClass("1", "2") content2 = DeserializableClass("3", "4") content = serializable_elist([content1, content2]) fj.write(content) content_r = fj.read() self.assertEqual(content, content_r)
def get_ebuild_data(pp, common_config={}, config={}): ebuild_data = {} package, version = pp['name'], pp['version'] if 'description' in pp: description = pp['description'] else: description = '' ebuild_data["realname"] = package ebuild_data["realversion"] = version ebuild_data["description"] = description ebuild_data["longdescription"] = description ebuild_data["homepage"] = pp["repository"]["url"] pkg_license = '' if "license" in pp: pkg_license = pp["license"] pkg_license = convert( [common_config, config], "licenses", pkg_license) ebuild_data["license"] = pkg_license ebuild_data["source_uri"] = pp["dist"]["tarball"] ebuild_data["md5"] = pp["dist"]["shasum"] if 'dependencies' in pp: ll = serializable_elist(separator="\n\t") for dep in pp["dependencies"].items(): # print(dep) ll.append(Dependency('dev-nodejs', dep[0])) #print(ll, '@@@@@') ebuild_data["dependencies"] = ll else: ebuild_data["dependencies"] = '' if 'devDependencies' in pp: ebuild_data["devDependencies"] = pp["devDependencies"] else: ebuild_data["devDependencies"] = '' return ebuild_data
def process_data(self, pkg_db, data, common_config, config): """ Process downloaded and parsed data and generate tree. Args: pkg_db: Package database. data: Dictionary with data, keys are file names. common_config; Backend config. config: Repository config. """ archive_contents = data['archive-contents'] repo_uri = config["repo_uri"] if sexpdata.car(archive_contents) != 1: raise SyncError('sync failed: ' \ + repo_uri + ' bad archive contents format') category = 'app-emacs' pkg_db.add_category(category) common_data = {'eclasses' : ['g-sorcery', 'gs-elpa'], 'maintainer' : [{'email' : '*****@*****.**', 'name' : 'Jauhien Piatlicki'}], 'homepage' : repo_uri, 'repo_uri' : repo_uri } pkg_db.set_common_data(category, common_data) PKG_INFO = 2 PKG_NAME = 0 INFO_VERSION = 0 INFO_DEPENDENCIES = 1 INFO_DESCRIPTION = 2 INFO_SRC_TYPE = 3 DEP_NAME = 0 #DEP_VERSION = 1 #we do not use it at the moment for entry in sexpdata.cdr(archive_contents): desc = entry[PKG_INFO].value() realname = entry[PKG_NAME].value() if self.in_config([common_config, config], "exclude", realname): continue pkg = Package("app-emacs", realname, '.'.join(map(str, desc[INFO_VERSION]))) source_type = desc[INFO_SRC_TYPE].value() allowed_ords = set(range(ord('a'), ord('z'))) \ | set(range(ord('A'), ord('Z'))) | \ set(range(ord('0'), ord('9'))) | set(list(map(ord, ['+', '_', '-', ' ', '.', '(', ')', '[', ']', '{', '}', ',']))) description = "".join([x for x in desc[INFO_DESCRIPTION] if ord(x) in allowed_ords]) deps = desc[INFO_DEPENDENCIES] #fix for crappy arhive-contents that have "No commentary." #in place of dependency if isinstance(deps, basestring): deps = [] dependencies = serializable_elist(separator="\n\t") for dep in deps: dep = self.convert_dependency([common_config, config], dep[DEP_NAME].value(), external = False) if dep: dependencies.append(dep) properties = {'source_type' : source_type, 'description' : description, 'dependencies' : dependencies, 'depend' : dependencies, 'rdepend' : dependencies, 'realname' : realname, 'longdescription' : description } pkg_db.add_package(pkg, properties)
def test_functionality(self): port = 8080 for fmt in SUPPORTED_FILE_FORMATS: sync_address = "127.0.0.1:" + str(port) + "/dummy.tar.gz" orig_tempdir = TemporaryDirectory() orig_path = os.path.join(orig_tempdir.name, "db") os.makedirs(orig_path) orig_db = PackageDB(orig_path, preferred_category_format=fmt) orig_db.add_category("app-test1") orig_db.add_category("app-test2") ebuild_data = { "test1": "tst1", "test2": "tst2", "test3": serializable_elist([ DeserializableClass("1", "2"), DeserializableClass("3", "4") ]) } common_data = { "common1": "cmn1", "common2": "cmn2", "common3": serializable_elist([ DeserializableClass("c1", "c2"), DeserializableClass("c3", "c4") ]) } packages = [ Package("app-test1", "test", "1"), Package("app-test1", "test", "2"), Package("app-test1", "test1", "1"), Package("app-test2", "test2", "1") ] for package in packages: orig_db.add_package(package, ebuild_data) orig_db.set_common_data("app-test1", common_data) full_data = dict(ebuild_data) full_data.update(common_data) orig_db.write() os.system("cd " + orig_tempdir.name + " && tar cvzf good.tar.gz db") os.system("echo invalid >> " + orig_tempdir.name + "/db/app-test1/packages." + fmt) os.system("cd " + orig_tempdir.name + " && tar cvzf dummy.tar.gz db") test_db = PackageDB(self.tempdir.name) self.assertRaises(SyncError, test_db.sync, sync_address) srv = Server(orig_tempdir.name, port=port) srv.start() try: self.assertRaises(IntegrityError, test_db.sync, sync_address) os.system("cd " + orig_tempdir.name + " && mv good.tar.gz dummy.tar.gz") test_db.sync(sync_address) finally: srv.shutdown() srv.join() test_db.read() self.assertEqual(orig_db.database, test_db.database) self.assertEqual(orig_db.get_common_data("app-test1"), test_db.get_common_data("app-test1")) self.assertEqual(orig_db.get_common_data("app-test2"), test_db.get_common_data("app-test2")) self.assertEqual(set(test_db.list_categories()), set(["app-test1", "app-test2"])) self.assertTrue(test_db.in_category("app-test1", "test")) self.assertFalse(test_db.in_category("app-test2", "test")) self.assertRaises(InvalidKeyError, test_db.in_category, "app-test3", "test") self.assertEqual(set(test_db.list_package_names("app-test1")), set(['test', 'test1'])) self.assertEqual( set(test_db.list_catpkg_names()), set(['app-test1/test', 'app-test1/test1', 'app-test2/test2'])) self.assertRaises(InvalidKeyError, test_db.list_package_versions, "invalid", "test") self.assertRaises(InvalidKeyError, test_db.list_package_versions, "app-test1", "invalid") self.assertEqual( set(test_db.list_package_versions("app-test1", "test")), set(['1', '2'])) self.assertEqual(set(test_db.list_all_packages()), set(packages)) self.assertEqual(test_db.get_package_description(packages[0]), full_data) self.assertRaises(KeyError, test_db.get_package_description, Package("invalid", "invalid", "1")) self.assertEqual(test_db.get_max_version("app-test1", "test"), "2") self.assertEqual(test_db.get_max_version("app-test1", "test1"), "1") self.assertRaises(InvalidKeyError, test_db.get_max_version, "invalid", "invalid") pkg_set = set(packages) for package, data in test_db: self.assertTrue(package in pkg_set) if package.category == "app-test1": self.assertEqual(data, full_data) else: self.assertEqual(data, ebuild_data) pkg_set.remove(package) self.assertTrue(not pkg_set) self.assertEqual(orig_db.database, test_db.database) port = port + 1
def process_data(self, pkg_db, data, common_config, config): """ Process parsed package data. """ category = "dev-python" pkg_db.add_category(category) common_data = {} common_data["eclasses"] = ['g-sorcery', 'gs-pypi'] common_data["maintainer"] = [{'email' : '*****@*****.**', 'name' : 'Jauhien Piatlicki'}] common_data["dependencies"] = serializable_elist(separator="\n\t") pkg_db.set_common_data(category, common_data) #todo: write filter functions allowed_ords_pkg = set(range(ord('a'), ord('z') + 1)) | set(range(ord('A'), ord('Z') + 1)) | \ set(range(ord('0'), ord('9') + 1)) | set(list(map(ord, ['+', '_', '-']))) allowed_ords_desc = set(range(ord('a'), ord('z') + 1)) | set(range(ord('A'), ord('Z') + 1)) | \ set(range(ord('0'), ord('9') + 1)) | set(list(map(ord, ['+', '_', '-', ' ', '.', '(', ')', '[', ']', '{', '}', ',']))) now = datetime.datetime.now() pseudoversion = "%04d%02d%02d" % (now.year, now.month, now.day) for (package, version), description in data["packages"]["index"].items(): pkg = package + "-" + version if not pkg in data["packages"]: continue pkg_data = data["packages"][pkg] if not pkg_data["files"] and not pkg_data["info"]: continue files_src_uri = "" md5 = "" if pkg_data["files"]: for file_entry in pkg_data["files"]: if file_entry["type"] == "\n Source\n ": files_src_uri = file_entry["url"] md5 = file_entry["md5"] break download_url = "" info = pkg_data["info"] if info: if "Download URL:" in info: download_url = info["Download URL:"] if download_url: source_uri = download_url #todo: find how to define src_uri else: source_uri = files_src_uri if not source_uri: continue homepage = "" pkg_license = "" py_versions = [] if info: if "Home Page:" in info: homepage = info["Home Page:"] categories = {} if "Categories" in info: categories = info["Categories"] if 'Programming Language' in categories: for entry in categories['Programming Language']: if entry == '2': py_versions.extend(['2_7']) elif entry == '3': py_versions.extend(['3_3', '3_4', '3_5']) elif entry == '2.6': py_versions.extend(['2_7']) elif entry == '2.7': py_versions.extend(['2_7']) elif entry == '3.2': py_versions.extend(['3_3']) elif entry == '3.3': py_versions.extend(['3_3']) elif entry == '3.4': py_versions.extend(['3_4']) elif entry == '3.5': py_versions.extend(['3_5']) if "License" in categories: pkg_license = categories["License"][-1] pkg_license = self.convert([common_config, config], "licenses", pkg_license) if not py_versions: py_versions = ['2_7', '3_3', '3_4', '3_5'] if len(py_versions) == 1: python_compat = '( python' + py_versions[0] + ' )' else: python_compat = '( python{' + py_versions[0] for ver in py_versions[1:]: python_compat += ',' + ver python_compat += '} )' filtered_package = "".join([x for x in package if ord(x) in allowed_ords_pkg]) description = "".join([x for x in description if ord(x) in allowed_ords_desc]) filtered_version = version match_object = re.match("(^[0-9]+[a-z]?$)|(^[0-9][0-9\.]+[0-9][a-z]?$)", filtered_version) if not match_object: filtered_version = pseudoversion ebuild_data = {} ebuild_data["realname"] = package ebuild_data["realversion"] = version ebuild_data["description"] = description ebuild_data["longdescription"] = description ebuild_data["homepage"] = homepage ebuild_data["license"] = pkg_license ebuild_data["source_uri"] = source_uri ebuild_data["md5"] = md5 ebuild_data["python_compat"] = python_compat pkg_db.add_package(Package(category, filtered_package, filtered_version), ebuild_data)
def test_functionality(self): port = 8080 for fmt in SUPPORTED_FILE_FORMATS: sync_address = "127.0.0.1:" + str(port) + "/dummy.tar.gz" orig_tempdir = TemporaryDirectory() orig_path = os.path.join(orig_tempdir.name, "db") os.makedirs(orig_path) orig_db = PackageDB(orig_path, preferred_category_format=fmt) orig_db.add_category("app-test1") orig_db.add_category("app-test2") ebuild_data = {"test1": "tst1", "test2": "tst2", "test3": serializable_elist([DeserializableClass("1", "2"), DeserializableClass("3", "4")])} common_data = {"common1": "cmn1", "common2": "cmn2", "common3": serializable_elist([DeserializableClass("c1", "c2"), DeserializableClass("c3", "c4")])} packages = [Package("app-test1", "test", "1"), Package("app-test1", "test", "2"), Package("app-test1", "test1", "1"), Package("app-test2", "test2", "1")] for package in packages: orig_db.add_package(package, ebuild_data) orig_db.set_common_data("app-test1", common_data) full_data = dict(ebuild_data) full_data.update(common_data) orig_db.write() os.system("cd " + orig_tempdir.name + " && tar cvzf good.tar.gz db") os.system("echo invalid >> " + orig_tempdir.name + "/db/app-test1/packages." + fmt) os.system("cd " + orig_tempdir.name + " && tar cvzf dummy.tar.gz db") test_db = PackageDB(self.tempdir.name) self.assertRaises(SyncError, test_db.sync, sync_address) srv = Server(orig_tempdir.name, port=port) srv.start() self.assertRaises(IntegrityError, test_db.sync, sync_address) os.system("cd " + orig_tempdir.name + " && mv good.tar.gz dummy.tar.gz") test_db.sync(sync_address) srv.shutdown() srv.join() test_db.read() self.assertEqual(orig_db.database, test_db.database) self.assertEqual(orig_db.get_common_data("app-test1"), test_db.get_common_data("app-test1")) self.assertEqual(orig_db.get_common_data("app-test2"), test_db.get_common_data("app-test2")) self.assertEqual(set(test_db.list_categories()), set(["app-test1", "app-test2"])) self.assertTrue(test_db.in_category("app-test1", "test")) self.assertFalse(test_db.in_category("app-test2", "test")) self.assertRaises(InvalidKeyError, test_db.in_category, "app-test3", "test") self.assertEqual(set(test_db.list_package_names("app-test1")), set(['test', 'test1'])) self.assertEqual(set(test_db.list_catpkg_names()),set(['app-test1/test', 'app-test1/test1', 'app-test2/test2'])) self.assertRaises(InvalidKeyError, test_db.list_package_versions, "invalid", "test") self.assertRaises(InvalidKeyError, test_db.list_package_versions, "app-test1", "invalid") self.assertEqual(set(test_db.list_package_versions("app-test1", "test")), set(['1', '2'])) self.assertEqual(set(test_db.list_all_packages()), set(packages)) self.assertEqual(test_db.get_package_description(packages[0]), full_data) self.assertRaises(KeyError, test_db.get_package_description, Package("invalid", "invalid", "1")) self.assertEqual(test_db.get_max_version("app-test1", "test"), "2") self.assertEqual(test_db.get_max_version("app-test1", "test1"), "1") self.assertRaises(InvalidKeyError, test_db.get_max_version, "invalid", "invalid") pkg_set = set(packages) for package, data in test_db: self.assertTrue(package in pkg_set) if package.category == "app-test1": self.assertEqual(data, full_data) else: self.assertEqual(data, ebuild_data) pkg_set.remove(package) self.assertTrue(not pkg_set) self.assertEqual(orig_db.database, test_db.database) port = port + 1
def process_data(self, pkg_db, data, common_config, config): """ Process parsed package data. """ category = "dev-python" pkg_db.add_category(category) common_data = {} common_data["eclasses"] = ['g-sorcery', 'gs-pypi'] common_data["maintainer"] = [{ 'email': '*****@*****.**', 'name': 'Jauhien Piatlicki' }] common_data["dependencies"] = serializable_elist(separator="\n\t") pkg_db.set_common_data(category, common_data) #todo: write filter functions allowed_ords_pkg = set(range(ord('a'), ord('z') + 1)) | set(range(ord('A'), ord('Z') + 1)) | \ set(range(ord('0'), ord('9') + 1)) | set(list(map(ord, ['+', '_', '-']))) allowed_ords_desc = set(range(ord('a'), ord('z') + 1)) | set(range(ord('A'), ord('Z') + 1)) | \ set(range(ord('0'), ord('9') + 1)) | set(list(map(ord, ['+', '_', '-', ' ', '.', '(', ')', '[', ']', '{', '}', ',']))) now = datetime.datetime.now() pseudoversion = "%04d%02d%02d" % (now.year, now.month, now.day) for (package, version), description in data["packages"]["index"].items(): pkg = package + "-" + version if not pkg in data["packages"]: continue pkg_data = data["packages"][pkg] if not pkg_data["files"] and not pkg_data["info"]: continue files_src_uri = "" md5 = "" if pkg_data["files"]: for file_entry in pkg_data["files"]: if file_entry["type"] == "\n Source\n ": files_src_uri = file_entry["url"] md5 = file_entry["md5"] break download_url = "" info = pkg_data["info"] if info: if "Download URL:" in info: download_url = info["Download URL:"] if download_url: source_uri = download_url #todo: find how to define src_uri else: source_uri = files_src_uri if not source_uri: continue homepage = "" pkg_license = "" py_versions = [] if info: if "Home Page:" in info: homepage = info["Home Page:"] categories = {} if "Categories" in info: categories = info["Categories"] if 'Programming Language' in categories: for entry in categories['Programming Language']: if entry == '2': py_versions.extend(['2_7']) elif entry == '3': py_versions.extend(['3_3', '3_4', '3_5']) elif entry == '2.6': py_versions.extend(['2_7']) elif entry == '2.7': py_versions.extend(['2_7']) elif entry == '3.2': py_versions.extend(['3_3']) elif entry == '3.3': py_versions.extend(['3_3']) elif entry == '3.4': py_versions.extend(['3_4']) elif entry == '3.5': py_versions.extend(['3_5']) if "License" in categories: pkg_license = categories["License"][-1] pkg_license = self.convert([common_config, config], "licenses", pkg_license) if not py_versions: py_versions = ['2_7', '3_3', '3_4', '3_5'] if len(py_versions) == 1: python_compat = '( python' + py_versions[0] + ' )' else: python_compat = '( python{' + py_versions[0] for ver in py_versions[1:]: python_compat += ',' + ver python_compat += '} )' filtered_package = "".join( [x for x in package if ord(x) in allowed_ords_pkg]) description = "".join( [x for x in description if ord(x) in allowed_ords_desc]) filtered_version = version match_object = re.match( "(^[0-9]+[a-z]?$)|(^[0-9][0-9\.]+[0-9][a-z]?$)", filtered_version) if not match_object: filtered_version = pseudoversion ebuild_data = {} ebuild_data["realname"] = package ebuild_data["realversion"] = version ebuild_data["description"] = description ebuild_data["longdescription"] = description ebuild_data["homepage"] = homepage ebuild_data["license"] = pkg_license ebuild_data["source_uri"] = source_uri ebuild_data["md5"] = md5 ebuild_data["python_compat"] = python_compat pkg_db.add_package( Package(category, filtered_package, filtered_version), ebuild_data)