def test(loop): output = None try: with open(os.devnull, 'rb', 0) as devnull: proc = loop.run_until_complete( asyncio.create_subprocess_exec(echo_binary, *args_tuple, stdin=devnull, stdout=stdout_pw, stderr=stdout_pw)) # This belongs exclusively to the subprocess now. stdout_pw.close() output = asyncio.ensure_future(reader(stdout_pr, loop=loop), loop=loop) self.assertEqual(loop.run_until_complete(proc.wait()), os.EX_OK) self.assertEqual( tuple(loop.run_until_complete(output).split()), args_tuple) finally: if output is not None and not output.done(): output.cancel() for f in files: f.close()
def _testPipeReader(self, master_fd, slave_fd, test_string): """ Use a poll loop to read data from a pipe and assert that the data written to the pipe is identical to the data read from the pipe. """ # WARNING: It is very important to use unbuffered mode here, # in order to avoid issue 5380 with python3. master_file = os.fdopen(master_fd, 'rb', 0) scheduler = global_event_loop() consumer = PipeReader(input_files={"producer": master_file}, _use_array=self._use_array, scheduler=scheduler) consumer.start() producer = scheduler.run_until_complete( asyncio.create_subprocess_exec("bash", "-c", self._echo_cmd % test_string, stdout=slave_fd, loop=scheduler)) os.close(slave_fd) scheduler.run_until_complete(producer.wait()) scheduler.run_until_complete(consumer.async_wait()) self.assertEqual(producer.returncode, os.EX_OK) self.assertEqual(consumer.returncode, os.EX_OK) return consumer.getvalue().decode('ascii', 'replace')
def test(loop): output = None try: proc = loop.run_until_complete( asyncio.create_subprocess_exec(cat_binary, stdin=stdin_pr, stdout=stdout_pw, stderr=stdout_pw)) # These belong exclusively to the subprocess now. stdout_pw.close() stdin_pr.close() output = asyncio.ensure_future(reader(stdout_pr, loop=loop), loop=loop) with ForkExecutor(loop=loop) as executor: writer = asyncio.ensure_future(loop.run_in_executor( executor, stdin_pw.write, stdin_data), loop=loop) # This belongs exclusively to the writer now. stdin_pw.close() loop.run_until_complete(writer) self.assertEqual(loop.run_until_complete(proc.wait()), os.EX_OK) self.assertEqual(loop.run_until_complete(output), stdin_data) finally: if output is not None and not output.done(): output.cancel() for f in files: f.close()
def _testPipeReader(self, master_fd, slave_fd, test_string): """ Use a poll loop to read data from a pipe and assert that the data written to the pipe is identical to the data read from the pipe. """ # WARNING: It is very important to use unbuffered mode here, # in order to avoid issue 5380 with python3. master_file = os.fdopen(master_fd, 'rb', 0) scheduler = global_event_loop() consumer = PipeReader( input_files={"producer" : master_file}, _use_array=self._use_array, scheduler=scheduler) consumer.start() producer = scheduler.run_until_complete(asyncio.create_subprocess_exec( "bash", "-c", self._echo_cmd % test_string, stdout=slave_fd, loop=scheduler)) os.close(slave_fd) scheduler.run_until_complete(producer.wait()) scheduler.run_until_complete(consumer.async_wait()) self.assertEqual(producer.returncode, os.EX_OK) self.assertEqual(consumer.returncode, os.EX_OK) return consumer.getvalue().decode('ascii', 'replace')
def _async_start(self): pipe_logger = None filter_proc = None try: log_input = None if self.log_path is not None: log_filter_file = self.log_filter_file if log_filter_file is not None: split_value = shlex_split(log_filter_file) log_filter_file = split_value if split_value else None if log_filter_file: filter_input, stdin = os.pipe() log_input, filter_output = os.pipe() try: filter_proc = yield asyncio.create_subprocess_exec( *log_filter_file, env=self.env, stdin=filter_input, stdout=filter_output, stderr=filter_output, loop=self.scheduler) except EnvironmentError: # Maybe the command is missing or broken somehow... os.close(filter_input) os.close(stdin) os.close(log_input) os.close(filter_output) else: self._stdin = os.fdopen(stdin, 'wb', 0) os.close(filter_input) os.close(filter_output) if self._stdin is None: # Since log_filter_file is unspecified or refers to a file # that is missing or broken somehow, create a pipe that # logs directly to pipe_logger. log_input, stdin = os.pipe() self._stdin = os.fdopen(stdin, 'wb', 0) # Set background=True so that pipe_logger does not log to stdout. pipe_logger = PipeLogger(background=True, scheduler=self.scheduler, input_fd=log_input, log_file_path=self.log_path) yield pipe_logger.async_start() except asyncio.CancelledError: if pipe_logger is not None and pipe_logger.poll() is None: pipe_logger.cancel() if filter_proc is not None and filter_proc.returncode is None: filter_proc.terminate() raise self._main_task = asyncio.ensure_future(self._main( pipe_logger, filter_proc=filter_proc), loop=self.scheduler) self._main_task.add_done_callback(self._main_exit)
def test(loop): with open(os.devnull, 'rb', 0) as devnull: proc = loop.run_until_complete( asyncio.create_subprocess_exec(echo_binary, *args_tuple, stdin=devnull, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)) self.assertEqual( tuple(loop.run_until_complete(proc.stdout.read()).split()), args_tuple) self.assertEqual(loop.run_until_complete(proc.wait()), os.EX_OK)
def test(loop): proc = loop.run_until_complete( asyncio.create_subprocess_exec(cat_binary, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)) # This buffers data when necessary to avoid blocking. proc.stdin.write(stdin_data) # Any buffered data is written asynchronously after the # close method is called. proc.stdin.close() self.assertEqual(loop.run_until_complete(proc.stdout.read()), stdin_data) self.assertEqual(loop.run_until_complete(proc.wait()), os.EX_OK)
def testEbuildFetch(self): user_config = { "make.conf": ( 'GENTOO_MIRRORS="{scheme}://{host}:{port}"', ), } distfiles = { 'bar': b'bar\n', 'foo': b'foo\n', } ebuilds = { 'dev-libs/A-1': { 'EAPI': '7', 'SRC_URI': '''{scheme}://{host}:{port}/distfiles/bar.txt -> bar {scheme}://{host}:{port}/distfiles/foo.txt -> foo''', }, } loop = SchedulerInterface(global_event_loop()) def run_async(func, *args, **kwargs): with ForkExecutor(loop=loop) as executor: return loop.run_until_complete(loop.run_in_executor(executor, functools.partial(func, *args, **kwargs))) scheme = 'http' host = '127.0.0.1' content = {} content['/distfiles/layout.conf'] = b'[structure]\n0=flat\n' for k, v in distfiles.items(): # mirror path content['/distfiles/{}'.format(k)] = v # upstream path content['/distfiles/{}.txt'.format(k)] = v with AsyncHTTPServer(host, content, loop) as server: ebuilds_subst = {} for cpv, metadata in ebuilds.items(): metadata = metadata.copy() metadata['SRC_URI'] = metadata['SRC_URI'].format( scheme=scheme, host=host, port=server.server_port) ebuilds_subst[cpv] = metadata user_config_subst = user_config.copy() for configname, configdata in user_config.items(): configdata_sub = [] for line in configdata: configdata_sub.append(line.format( scheme=scheme, host=host, port=server.server_port)) user_config_subst[configname] = tuple(configdata_sub) playground = ResolverPlayground(ebuilds=ebuilds_subst, distfiles=distfiles, user_config=user_config_subst) ro_distdir = tempfile.mkdtemp() eubin = os.path.join(playground.eprefix, "usr", "bin") try: fetchcommand = portage.util.shlex_split(playground.settings['FETCHCOMMAND']) fetch_bin = portage.process.find_binary(fetchcommand[0]) if fetch_bin is None: self.skipTest('FETCHCOMMAND not found: {}'.format(playground.settings['FETCHCOMMAND'])) os.symlink(fetch_bin, os.path.join(eubin, os.path.basename(fetch_bin))) resumecommand = portage.util.shlex_split(playground.settings['RESUMECOMMAND']) resume_bin = portage.process.find_binary(resumecommand[0]) if resume_bin is None: self.skipTest('RESUMECOMMAND not found: {}'.format(playground.settings['RESUMECOMMAND'])) if resume_bin != fetch_bin: os.symlink(resume_bin, os.path.join(eubin, os.path.basename(resume_bin))) root_config = playground.trees[playground.eroot]['root_config'] portdb = root_config.trees["porttree"].dbapi settings = config(clone=playground.settings) # Demonstrate that fetch preserves a stale file in DISTDIR when no digests are given. foo_uri = {'foo': ('{scheme}://{host}:{port}/distfiles/foo'.format(scheme=scheme, host=host, port=server.server_port),)} foo_path = os.path.join(settings['DISTDIR'], 'foo') foo_stale_content = b'stale content\n' with open(foo_path, 'wb') as f: f.write(b'stale content\n') self.assertTrue(bool(run_async(fetch, foo_uri, settings, try_mirrors=False))) with open(foo_path, 'rb') as f: self.assertEqual(f.read(), foo_stale_content) with open(foo_path, 'rb') as f: self.assertNotEqual(f.read(), distfiles['foo']) # Use force=True to update the stale file. self.assertTrue(bool(run_async(fetch, foo_uri, settings, try_mirrors=False, force=True))) with open(foo_path, 'rb') as f: self.assertEqual(f.read(), distfiles['foo']) # Test force=True with FEATURES=skiprocheck, using read-only DISTDIR. # FETCHCOMMAND is set to temporarily chmod +w DISTDIR. Note that # FETCHCOMMAND must perform atomic rename itself due to read-only # DISTDIR. with open(foo_path, 'wb') as f: f.write(b'stale content\n') orig_fetchcommand = settings['FETCHCOMMAND'] orig_distdir_mode = os.stat(settings['DISTDIR']).st_mode temp_fetchcommand = os.path.join(eubin, 'fetchcommand') with open(temp_fetchcommand, 'w') as f: f.write(""" set -e URI=$1 DISTDIR=$2 FILE=$3 trap 'chmod a-w "${DISTDIR}"' EXIT chmod ug+w "${DISTDIR}" %s mv -f "${DISTDIR}/${FILE}.__download__" "${DISTDIR}/${FILE}" """ % orig_fetchcommand.replace('${FILE}', '${FILE}.__download__')) settings['FETCHCOMMAND'] = '"%s" "%s" "${URI}" "${DISTDIR}" "${FILE}"' % (BASH_BINARY, temp_fetchcommand) settings.features.add('skiprocheck') settings.features.remove('distlocks') os.chmod(settings['DISTDIR'], 0o555) try: self.assertTrue(bool(run_async(fetch, foo_uri, settings, try_mirrors=False, force=True))) finally: settings['FETCHCOMMAND'] = orig_fetchcommand os.chmod(settings['DISTDIR'], orig_distdir_mode) settings.features.remove('skiprocheck') settings.features.add('distlocks') os.unlink(temp_fetchcommand) with open(foo_path, 'rb') as f: self.assertEqual(f.read(), distfiles['foo']) # Test emirrordist invocation. emirrordist_cmd = (portage._python_interpreter, '-b', '-Wd', os.path.join(self.bindir, 'emirrordist'), '--distfiles', settings['DISTDIR'], '--config-root', settings['EPREFIX'], '--repositories-configuration', settings.repositories.config_string(), '--repo', 'test_repo', '--mirror') env = settings.environ() env['PYTHONPATH'] = ':'.join( filter(None, [PORTAGE_PYM_PATH] + os.environ.get('PYTHONPATH', '').split(':'))) for k in distfiles: os.unlink(os.path.join(settings['DISTDIR'], k)) proc = loop.run_until_complete(asyncio.create_subprocess_exec(*emirrordist_cmd, env=env)) self.assertEqual(loop.run_until_complete(proc.wait()), 0) for k in distfiles: with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) # Tests only work with one ebuild at a time, so the config # pool only needs a single config instance. class config_pool: @staticmethod def allocate(): return settings @staticmethod def deallocate(settings): pass def async_fetch(pkg, ebuild_path): fetcher = EbuildFetcher(config_pool=config_pool, ebuild_path=ebuild_path, fetchonly=False, fetchall=True, pkg=pkg, scheduler=loop) fetcher.start() return fetcher.async_wait() for cpv in ebuilds: metadata = dict(zip(Package.metadata_keys, portdb.aux_get(cpv, Package.metadata_keys))) pkg = Package(built=False, cpv=cpv, installed=False, metadata=metadata, root_config=root_config, type_name='ebuild') settings.setcpv(pkg) ebuild_path = portdb.findname(pkg.cpv) portage.doebuild_environment(ebuild_path, 'fetch', settings=settings, db=portdb) # Test good files in DISTDIR for k in settings['AA'].split(): os.stat(os.path.join(settings['DISTDIR'], k)) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) # Test digestgen with fetch os.unlink(os.path.join(os.path.dirname(ebuild_path), 'Manifest')) for k in settings['AA'].split(): os.unlink(os.path.join(settings['DISTDIR'], k)) with ForkExecutor(loop=loop) as executor: self.assertTrue(bool(loop.run_until_complete( loop.run_in_executor(executor, functools.partial( digestgen, mysettings=settings, myportdb=portdb))))) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) # Test missing files in DISTDIR for k in settings['AA'].split(): os.unlink(os.path.join(settings['DISTDIR'], k)) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) # Test empty files in DISTDIR for k in settings['AA'].split(): file_path = os.path.join(settings['DISTDIR'], k) with open(file_path, 'wb') as f: pass self.assertEqual(os.stat(file_path).st_size, 0) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) # Test non-empty files containing null bytes in DISTDIR for k in settings['AA'].split(): file_path = os.path.join(settings['DISTDIR'], k) with open(file_path, 'wb') as f: f.write(len(distfiles[k]) * b'\0') self.assertEqual(os.stat(file_path).st_size, len(distfiles[k])) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) # Test PORTAGE_RO_DISTDIRS settings['PORTAGE_RO_DISTDIRS'] = '"{}"'.format(ro_distdir) orig_fetchcommand = settings['FETCHCOMMAND'] orig_resumecommand = settings['RESUMECOMMAND'] try: settings['FETCHCOMMAND'] = settings['RESUMECOMMAND'] = '' for k in settings['AA'].split(): file_path = os.path.join(settings['DISTDIR'], k) os.rename(file_path, os.path.join(ro_distdir, k)) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): file_path = os.path.join(settings['DISTDIR'], k) self.assertTrue(os.path.islink(file_path)) with open(file_path, 'rb') as f: self.assertEqual(f.read(), distfiles[k]) os.unlink(file_path) finally: settings.pop('PORTAGE_RO_DISTDIRS') settings['FETCHCOMMAND'] = orig_fetchcommand settings['RESUMECOMMAND'] = orig_resumecommand # Test local filesystem in GENTOO_MIRRORS orig_mirrors = settings['GENTOO_MIRRORS'] orig_fetchcommand = settings['FETCHCOMMAND'] try: settings['GENTOO_MIRRORS'] = ro_distdir settings['FETCHCOMMAND'] = settings['RESUMECOMMAND'] = '' self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) finally: settings['GENTOO_MIRRORS'] = orig_mirrors settings['FETCHCOMMAND'] = orig_fetchcommand settings['RESUMECOMMAND'] = orig_resumecommand # Test readonly DISTDIR orig_distdir_mode = os.stat(settings['DISTDIR']).st_mode try: os.chmod(settings['DISTDIR'], 0o555) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) finally: os.chmod(settings['DISTDIR'], orig_distdir_mode) # Test parallel-fetch mode settings['PORTAGE_PARALLEL_FETCHONLY'] = '1' try: self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) for k in settings['AA'].split(): os.unlink(os.path.join(settings['DISTDIR'], k)) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) finally: settings.pop('PORTAGE_PARALLEL_FETCHONLY') # Test RESUMECOMMAND orig_resume_min_size = settings['PORTAGE_FETCH_RESUME_MIN_SIZE'] try: settings['PORTAGE_FETCH_RESUME_MIN_SIZE'] = '2' for k in settings['AA'].split(): file_path = os.path.join(settings['DISTDIR'], k) os.unlink(file_path) with open(file_path + _download_suffix, 'wb') as f: f.write(distfiles[k][:2]) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) finally: settings['PORTAGE_FETCH_RESUME_MIN_SIZE'] = orig_resume_min_size # Test readonly DISTDIR + skiprocheck, with FETCHCOMMAND set to temporarily chmod DISTDIR orig_fetchcommand = settings['FETCHCOMMAND'] orig_distdir_mode = os.stat(settings['DISTDIR']).st_mode for k in settings['AA'].split(): os.unlink(os.path.join(settings['DISTDIR'], k)) try: os.chmod(settings['DISTDIR'], 0o555) settings['FETCHCOMMAND'] = '"%s" -c "chmod ug+w \\"${DISTDIR}\\"; %s; status=\\$?; chmod a-w \\"${DISTDIR}\\"; exit \\$status"' % (BASH_BINARY, orig_fetchcommand.replace('"', '\\"')) settings.features.add('skiprocheck') settings.features.remove('distlocks') self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) finally: settings['FETCHCOMMAND'] = orig_fetchcommand os.chmod(settings['DISTDIR'], orig_distdir_mode) settings.features.remove('skiprocheck') settings.features.add('distlocks') finally: shutil.rmtree(ro_distdir) playground.cleanup()
def _async_test_simple(self, loop, playground, metadata_xml_files): debug = playground.debug settings = playground.settings eprefix = settings["EPREFIX"] eroot = settings["EROOT"] trees = playground.trees portdb = trees[eroot]["porttree"].dbapi test_repo_location = settings.repositories["test_repo"].location var_cache_edb = os.path.join(eprefix, "var", "cache", "edb") cachedir = os.path.join(var_cache_edb, "dep") cachedir_pregen = os.path.join(test_repo_location, "metadata", "md5-cache") portage_python = portage._python_interpreter dispatch_conf_cmd = (portage_python, "-b", "-Wd", os.path.join(self.sbindir, "dispatch-conf")) ebuild_cmd = (portage_python, "-b", "-Wd", os.path.join(self.bindir, "ebuild")) egencache_cmd = (portage_python, "-b", "-Wd", os.path.join(self.bindir, "egencache"), "--repo", "test_repo", "--repositories-configuration", settings.repositories.config_string()) emerge_cmd = (portage_python, "-b", "-Wd", os.path.join(self.bindir, "emerge")) emaint_cmd = (portage_python, "-b", "-Wd", os.path.join(self.sbindir, "emaint")) env_update_cmd = (portage_python, "-b", "-Wd", os.path.join(self.sbindir, "env-update")) etc_update_cmd = (BASH_BINARY, os.path.join(self.sbindir, "etc-update")) fixpackages_cmd = (portage_python, "-b", "-Wd", os.path.join(self.sbindir, "fixpackages")) portageq_cmd = (portage_python, "-b", "-Wd", os.path.join(self.bindir, "portageq")) quickpkg_cmd = (portage_python, "-b", "-Wd", os.path.join(self.bindir, "quickpkg")) regenworld_cmd = (portage_python, "-b", "-Wd", os.path.join(self.sbindir, "regenworld")) rm_binary = find_binary("rm") self.assertEqual(rm_binary is None, False, "rm command not found") rm_cmd = (rm_binary, ) egencache_extra_args = [] if self._have_python_xml(): egencache_extra_args.append("--update-use-local-desc") test_ebuild = portdb.findname("dev-libs/A-1") self.assertFalse(test_ebuild is None) cross_prefix = os.path.join(eprefix, "cross_prefix") cross_root = os.path.join(eprefix, "cross_root") cross_eroot = os.path.join(cross_root, eprefix.lstrip(os.sep)) binhost_dir = os.path.join(eprefix, "binhost") binhost_address = '127.0.0.1' binhost_remote_path = '/binhost' binhost_server = AsyncHTTPServer( binhost_address, BinhostContentMap(binhost_remote_path, binhost_dir), loop).__enter__() binhost_uri = 'http://{address}:{port}{path}'.format( address=binhost_address, port=binhost_server.server_port, path=binhost_remote_path) test_commands = ( emerge_cmd + ("--usepkgonly", "--root", cross_root, "--quickpkg-direct=y", "dev-libs/A"), env_update_cmd, portageq_cmd + ("envvar", "-v", "CONFIG_PROTECT", "EROOT", "PORTAGE_CONFIGROOT", "PORTAGE_TMPDIR", "USERLAND"), etc_update_cmd, dispatch_conf_cmd, emerge_cmd + ("--version",), emerge_cmd + ("--info",), emerge_cmd + ("--info", "--verbose"), emerge_cmd + ("--list-sets",), emerge_cmd + ("--check-news",), rm_cmd + ("-rf", cachedir), rm_cmd + ("-rf", cachedir_pregen), emerge_cmd + ("--regen",), rm_cmd + ("-rf", cachedir), ({"FEATURES" : "metadata-transfer"},) + \ emerge_cmd + ("--regen",), rm_cmd + ("-rf", cachedir), ({"FEATURES" : "metadata-transfer"},) + \ emerge_cmd + ("--regen",), rm_cmd + ("-rf", cachedir), egencache_cmd + ("--update",) + tuple(egencache_extra_args), ({"FEATURES" : "metadata-transfer"},) + \ emerge_cmd + ("--metadata",), rm_cmd + ("-rf", cachedir), ({"FEATURES" : "metadata-transfer"},) + \ emerge_cmd + ("--metadata",), emerge_cmd + ("--metadata",), rm_cmd + ("-rf", cachedir), emerge_cmd + ("--oneshot", "virtual/foo"), lambda: self.assertFalse(os.path.exists( os.path.join(pkgdir, "virtual", "foo-0.tbz2"))), ({"FEATURES" : "unmerge-backup"},) + \ emerge_cmd + ("--unmerge", "virtual/foo"), lambda: self.assertTrue(os.path.exists( os.path.join(pkgdir, "virtual", "foo-0.tbz2"))), emerge_cmd + ("--pretend", "dev-libs/A"), ebuild_cmd + (test_ebuild, "manifest", "clean", "package", "merge"), emerge_cmd + ("--pretend", "--tree", "--complete-graph", "dev-libs/A"), emerge_cmd + ("-p", "dev-libs/B"), emerge_cmd + ("-p", "--newrepo", "dev-libs/B"), emerge_cmd + ("-B", "dev-libs/B",), emerge_cmd + ("--oneshot", "--usepkg", "dev-libs/B",), # trigger clean prior to pkg_pretend as in bug #390711 ebuild_cmd + (test_ebuild, "unpack"), emerge_cmd + ("--oneshot", "dev-libs/A",), emerge_cmd + ("--noreplace", "dev-libs/A",), emerge_cmd + ("--config", "dev-libs/A",), emerge_cmd + ("--info", "dev-libs/A", "dev-libs/B"), emerge_cmd + ("--pretend", "--depclean", "--verbose", "dev-libs/B"), emerge_cmd + ("--pretend", "--depclean",), emerge_cmd + ("--depclean",), quickpkg_cmd + ("--include-config", "y", "dev-libs/A",), # Test bug #523684, where a file renamed or removed by the # admin forces replacement files to be merged with config # protection. lambda: self.assertEqual(0, len(list(find_updated_config_files(eroot, shlex_split(settings["CONFIG_PROTECT"]))))), lambda: os.unlink(os.path.join(eprefix, "etc", "A-0")), emerge_cmd + ("--usepkgonly", "dev-libs/A"), lambda: self.assertEqual(1, len(list(find_updated_config_files(eroot, shlex_split(settings["CONFIG_PROTECT"]))))), emaint_cmd + ("--check", "all"), emaint_cmd + ("--fix", "all"), fixpackages_cmd, regenworld_cmd, portageq_cmd + ("match", eroot, "dev-libs/A"), portageq_cmd + ("best_visible", eroot, "dev-libs/A"), portageq_cmd + ("best_visible", eroot, "binary", "dev-libs/A"), portageq_cmd + ("contents", eroot, "dev-libs/A-1"), portageq_cmd + ("metadata", eroot, "ebuild", "dev-libs/A-1", "EAPI", "IUSE", "RDEPEND"), portageq_cmd + ("metadata", eroot, "binary", "dev-libs/A-1", "EAPI", "USE", "RDEPEND"), portageq_cmd + ("metadata", eroot, "installed", "dev-libs/A-1", "EAPI", "USE", "RDEPEND"), portageq_cmd + ("owners", eroot, eroot + "usr"), emerge_cmd + ("-p", eroot + "usr"), emerge_cmd + ("-p", "--unmerge", "-q", eroot + "usr"), emerge_cmd + ("--unmerge", "--quiet", "dev-libs/A"), emerge_cmd + ("-C", "--quiet", "dev-libs/B"), # If EMERGE_DEFAULT_OPTS contains --autounmask=n, then --autounmask # must be specified with --autounmask-continue. ({"EMERGE_DEFAULT_OPTS" : "--autounmask=n"},) + \ emerge_cmd + ("--autounmask", "--autounmask-continue", "dev-libs/C",), # Verify that the above --autounmask-continue command caused # USE=flag to be applied correctly to dev-libs/D. portageq_cmd + ("match", eroot, "dev-libs/D[flag]"), # Test cross-prefix usage, including chpathtool for binpkgs. # EAPI 7 ({"EPREFIX" : cross_prefix},) + \ emerge_cmd + ("dev-libs/C",), ({"EPREFIX" : cross_prefix},) + \ portageq_cmd + ("has_version", cross_prefix, "dev-libs/C"), ({"EPREFIX" : cross_prefix},) + \ portageq_cmd + ("has_version", cross_prefix, "dev-libs/D"), ({"ROOT": cross_root},) + emerge_cmd + ("dev-libs/D",), portageq_cmd + ("has_version", cross_eroot, "dev-libs/D"), # EAPI 5 ({"EPREFIX" : cross_prefix},) + \ emerge_cmd + ("--usepkgonly", "dev-libs/A"), ({"EPREFIX" : cross_prefix},) + \ portageq_cmd + ("has_version", cross_prefix, "dev-libs/A"), ({"EPREFIX" : cross_prefix},) + \ portageq_cmd + ("has_version", cross_prefix, "dev-libs/B"), ({"EPREFIX" : cross_prefix},) + \ emerge_cmd + ("-C", "--quiet", "dev-libs/B"), ({"EPREFIX" : cross_prefix},) + \ emerge_cmd + ("-C", "--quiet", "dev-libs/A"), ({"EPREFIX" : cross_prefix},) + \ emerge_cmd + ("dev-libs/A",), ({"EPREFIX" : cross_prefix},) + \ portageq_cmd + ("has_version", cross_prefix, "dev-libs/A"), ({"EPREFIX" : cross_prefix},) + \ portageq_cmd + ("has_version", cross_prefix, "dev-libs/B"), # Test ROOT support ({"ROOT": cross_root},) + emerge_cmd + ("dev-libs/B",), portageq_cmd + ("has_version", cross_eroot, "dev-libs/B"), ) # Test binhost support if FETCHCOMMAND is available. fetchcommand = portage.util.shlex_split( playground.settings['FETCHCOMMAND']) fetch_bin = portage.process.find_binary(fetchcommand[0]) if fetch_bin is not None: test_commands = test_commands + ( lambda: os.rename(pkgdir, binhost_dir), ({"PORTAGE_BINHOST": binhost_uri},) + \ emerge_cmd + ("-e", "--getbinpkgonly", "dev-libs/A"), lambda: shutil.rmtree(pkgdir), lambda: os.rename(binhost_dir, pkgdir), ) distdir = playground.distdir pkgdir = playground.pkgdir fake_bin = os.path.join(eprefix, "bin") portage_tmpdir = os.path.join(eprefix, "var", "tmp", "portage") profile_path = settings.profile_path user_config_dir = os.path.join(os.sep, eprefix, USER_CONFIG_PATH) path = os.environ.get("PATH") if path is not None and not path.strip(): path = None if path is None: path = "" else: path = ":" + path path = fake_bin + path pythonpath = os.environ.get("PYTHONPATH") if pythonpath is not None and not pythonpath.strip(): pythonpath = None if pythonpath is not None and \ pythonpath.split(":")[0] == PORTAGE_PYM_PATH: pass else: if pythonpath is None: pythonpath = "" else: pythonpath = ":" + pythonpath pythonpath = PORTAGE_PYM_PATH + pythonpath env = { "PORTAGE_OVERRIDE_EPREFIX": eprefix, "CLEAN_DELAY": "0", "DISTDIR": distdir, "EMERGE_WARNING_DELAY": "0", "INFODIR": "", "INFOPATH": "", "PATH": path, "PKGDIR": pkgdir, "PORTAGE_INST_GID": str(portage.data.portage_gid), "PORTAGE_INST_UID": str(portage.data.portage_uid), "PORTAGE_PYTHON": portage_python, "PORTAGE_REPOSITORIES": settings.repositories.config_string(), "PORTAGE_TMPDIR": portage_tmpdir, "PORTAGE_LOGDIR": portage_tmpdir, "PYTHONDONTWRITEBYTECODE": os.environ.get("PYTHONDONTWRITEBYTECODE", ""), "PYTHONPATH": pythonpath, "__PORTAGE_TEST_PATH_OVERRIDE": fake_bin, } if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \ os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"] updates_dir = os.path.join(test_repo_location, "profiles", "updates") dirs = [ cachedir, cachedir_pregen, cross_eroot, cross_prefix, distdir, fake_bin, portage_tmpdir, updates_dir, user_config_dir, var_cache_edb ] etc_symlinks = ("dispatch-conf.conf", "etc-update.conf") # Override things that may be unavailable, or may have portability # issues when running tests in exotic environments. # prepstrip - bug #447810 (bash read builtin EINTR problem) true_symlinks = ["find", "prepstrip", "sed", "scanelf"] true_binary = find_binary("true") self.assertEqual(true_binary is None, False, "true command not found") try: for d in dirs: ensure_dirs(d) for x in true_symlinks: os.symlink(true_binary, os.path.join(fake_bin, x)) for x in etc_symlinks: os.symlink(os.path.join(self.cnf_etc_path, x), os.path.join(eprefix, "etc", x)) with open(os.path.join(var_cache_edb, "counter"), 'wb') as f: f.write(b"100") # non-empty system set keeps --depclean quiet with open(os.path.join(profile_path, "packages"), 'w') as f: f.write("*dev-libs/token-system-pkg") for cp, xml_data in metadata_xml_files: with open(os.path.join(test_repo_location, cp, "metadata.xml"), 'w') as f: f.write(playground.metadata_xml_template % xml_data) with open(os.path.join(updates_dir, "1Q-2010"), 'w') as f: f.write(""" slotmove =app-doc/pms-3 2 3 move dev-util/git dev-vcs/git """) if debug: # The subprocess inherits both stdout and stderr, for # debugging purposes. stdout = None else: # The subprocess inherits stderr so that any warnings # triggered by python -Wd will be visible. stdout = subprocess.PIPE for args in test_commands: if hasattr(args, '__call__'): args() continue if isinstance(args[0], dict): local_env = env.copy() local_env.update(args[0]) args = args[1:] else: local_env = env proc = yield asyncio.create_subprocess_exec(*args, env=local_env, stderr=None, stdout=stdout) if debug: yield proc.wait() else: output, _err = yield proc.communicate() yield proc.wait() if proc.returncode != os.EX_OK: for line in output: sys.stderr.write(_unicode_decode(line)) self.assertEqual(os.EX_OK, proc.returncode, "emerge failed with args %s" % (args, )) finally: binhost_server.__exit__(None, None, None) playground.cleanup()
def _testEbuildFetch( self, loop, scheme, host, orig_distfiles, ebuilds, content, server, playground, ro_distdir, ): mirror_layouts = ( ( "[structure]", "0=filename-hash BLAKE2B 8", "1=flat", ), ( "[structure]", "1=filename-hash BLAKE2B 8", "0=flat", ), ( "[structure]", "0=content-hash SHA512 8:8:8", "1=flat", ), ) fetchcommand = portage.util.shlex_split( playground.settings["FETCHCOMMAND"]) fetch_bin = portage.process.find_binary(fetchcommand[0]) if fetch_bin is None: self.skipTest("FETCHCOMMAND not found: {}".format( playground.settings["FETCHCOMMAND"])) eubin = os.path.join(playground.eprefix, "usr", "bin") os.symlink(fetch_bin, os.path.join(eubin, os.path.basename(fetch_bin))) resumecommand = portage.util.shlex_split( playground.settings["RESUMECOMMAND"]) resume_bin = portage.process.find_binary(resumecommand[0]) if resume_bin is None: self.skipTest("RESUMECOMMAND not found: {}".format( playground.settings["RESUMECOMMAND"])) if resume_bin != fetch_bin: os.symlink(resume_bin, os.path.join(eubin, os.path.basename(resume_bin))) root_config = playground.trees[playground.eroot]["root_config"] portdb = root_config.trees["porttree"].dbapi def run_async(func, *args, **kwargs): with ForkExecutor(loop=loop) as executor: return loop.run_until_complete( loop.run_in_executor( executor, functools.partial(func, *args, **kwargs))) for layout_lines in mirror_layouts: settings = config(clone=playground.settings) layout_data = "".join("{}\n".format(line) for line in layout_lines) mirror_conf = MirrorLayoutConfig() mirror_conf.read_from_file(io.StringIO(layout_data)) layouts = mirror_conf.get_all_layouts() content["/distfiles/layout.conf"] = layout_data.encode("utf8") distfiles = {} for k, v in orig_distfiles.items(): filename = DistfileName( k, digests=dict((algo, checksum_str(v, hashname=algo)) for algo in MANIFEST2_HASH_DEFAULTS), ) distfiles[filename] = v # mirror path for layout in layouts: content["/distfiles/" + layout.get_path(filename)] = v # upstream path content["/distfiles/{}.txt".format(k)] = v shutil.rmtree(settings["DISTDIR"]) os.makedirs(settings["DISTDIR"]) with open(os.path.join(settings["DISTDIR"], "layout.conf"), "wt") as f: f.write(layout_data) if any( isinstance(layout, ContentHashLayout) for layout in layouts): content_db = os.path.join(playground.eprefix, "var/db/emirrordist/content.db") os.makedirs(os.path.dirname(content_db), exist_ok=True) try: os.unlink(content_db) except OSError: pass else: content_db = None # Demonstrate that fetch preserves a stale file in DISTDIR when no digests are given. foo_uri = { "foo": ("{scheme}://{host}:{port}/distfiles/foo".format( scheme=scheme, host=host, port=server.server_port), ) } foo_path = os.path.join(settings["DISTDIR"], "foo") foo_stale_content = b"stale content\n" with open(foo_path, "wb") as f: f.write(b"stale content\n") self.assertTrue( bool(run_async(fetch, foo_uri, settings, try_mirrors=False))) with open(foo_path, "rb") as f: self.assertEqual(f.read(), foo_stale_content) with open(foo_path, "rb") as f: self.assertNotEqual(f.read(), distfiles["foo"]) # Use force=True to update the stale file. self.assertTrue( bool( run_async(fetch, foo_uri, settings, try_mirrors=False, force=True))) with open(foo_path, "rb") as f: self.assertEqual(f.read(), distfiles["foo"]) # Test force=True with FEATURES=skiprocheck, using read-only DISTDIR. # FETCHCOMMAND is set to temporarily chmod +w DISTDIR. Note that # FETCHCOMMAND must perform atomic rename itself due to read-only # DISTDIR. with open(foo_path, "wb") as f: f.write(b"stale content\n") orig_fetchcommand = settings["FETCHCOMMAND"] orig_distdir_mode = os.stat(settings["DISTDIR"]).st_mode temp_fetchcommand = os.path.join(eubin, "fetchcommand") with open(temp_fetchcommand, "w") as f: f.write(""" set -e URI=$1 DISTDIR=$2 FILE=$3 trap 'chmod a-w "${DISTDIR}"' EXIT chmod ug+w "${DISTDIR}" %s mv -f "${DISTDIR}/${FILE}.__download__" "${DISTDIR}/${FILE}" """ % orig_fetchcommand.replace("${FILE}", "${FILE}.__download__")) settings[ "FETCHCOMMAND"] = '"%s" "%s" "${URI}" "${DISTDIR}" "${FILE}"' % ( BASH_BINARY, temp_fetchcommand, ) settings.features.add("skiprocheck") settings.features.remove("distlocks") os.chmod(settings["DISTDIR"], 0o555) try: self.assertTrue( bool( run_async(fetch, foo_uri, settings, try_mirrors=False, force=True))) finally: settings["FETCHCOMMAND"] = orig_fetchcommand os.chmod(settings["DISTDIR"], orig_distdir_mode) settings.features.remove("skiprocheck") settings.features.add("distlocks") os.unlink(temp_fetchcommand) with open(foo_path, "rb") as f: self.assertEqual(f.read(), distfiles["foo"]) # Test emirrordist invocation. emirrordist_cmd = ( portage._python_interpreter, "-b", "-Wd", os.path.join(self.bindir, "emirrordist"), "--distfiles", settings["DISTDIR"], "--config-root", settings["EPREFIX"], "--delete", "--repositories-configuration", settings.repositories.config_string(), "--repo", "test_repo", "--mirror", ) if content_db is not None: emirrordist_cmd = emirrordist_cmd + ( "--content-db", content_db, ) env = settings.environ() env["PYTHONPATH"] = ":".join( filter( None, [PORTAGE_PYM_PATH] + os.environ.get("PYTHONPATH", "").split(":"), )) for k in distfiles: try: os.unlink(os.path.join(settings["DISTDIR"], k)) except OSError: pass proc = loop.run_until_complete( asyncio.create_subprocess_exec(*emirrordist_cmd, env=env)) self.assertEqual(loop.run_until_complete(proc.wait()), 0) for k in distfiles: with open( os.path.join(settings["DISTDIR"], layouts[0].get_path(k)), "rb") as f: self.assertEqual(f.read(), distfiles[k]) if content_db is not None: loop.run_until_complete( self._test_content_db( emirrordist_cmd, env, layouts, content_db, distfiles, settings, portdb, )) # Tests only work with one ebuild at a time, so the config # pool only needs a single config instance. class config_pool: @staticmethod def allocate(): return settings @staticmethod def deallocate(settings): pass def async_fetch(pkg, ebuild_path): fetcher = EbuildFetcher( config_pool=config_pool, ebuild_path=ebuild_path, fetchonly=False, fetchall=True, pkg=pkg, scheduler=loop, ) fetcher.start() return fetcher.async_wait() for cpv in ebuilds: metadata = dict( zip( Package.metadata_keys, portdb.aux_get(cpv, Package.metadata_keys), )) pkg = Package( built=False, cpv=cpv, installed=False, metadata=metadata, root_config=root_config, type_name="ebuild", ) settings.setcpv(pkg) ebuild_path = portdb.findname(pkg.cpv) portage.doebuild_environment(ebuild_path, "fetch", settings=settings, db=portdb) # Test good files in DISTDIR for k in settings["AA"].split(): os.stat(os.path.join(settings["DISTDIR"], k)) self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) # Test digestgen with fetch os.unlink( os.path.join(os.path.dirname(ebuild_path), "Manifest")) for k in settings["AA"].split(): os.unlink(os.path.join(settings["DISTDIR"], k)) with ForkExecutor(loop=loop) as executor: self.assertTrue( bool( loop.run_until_complete( loop.run_in_executor( executor, functools.partial(digestgen, mysettings=settings, myportdb=portdb), )))) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) # Test missing files in DISTDIR for k in settings["AA"].split(): os.unlink(os.path.join(settings["DISTDIR"], k)) self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) # Test empty files in DISTDIR for k in settings["AA"].split(): file_path = os.path.join(settings["DISTDIR"], k) with open(file_path, "wb") as f: pass self.assertEqual(os.stat(file_path).st_size, 0) self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) # Test non-empty files containing null bytes in DISTDIR for k in settings["AA"].split(): file_path = os.path.join(settings["DISTDIR"], k) with open(file_path, "wb") as f: f.write(len(distfiles[k]) * b"\0") self.assertEqual( os.stat(file_path).st_size, len(distfiles[k])) self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) # Test PORTAGE_RO_DISTDIRS settings["PORTAGE_RO_DISTDIRS"] = '"{}"'.format(ro_distdir) orig_fetchcommand = settings["FETCHCOMMAND"] orig_resumecommand = settings["RESUMECOMMAND"] try: settings["FETCHCOMMAND"] = settings["RESUMECOMMAND"] = "" for k in settings["AA"].split(): file_path = os.path.join(settings["DISTDIR"], k) os.rename(file_path, os.path.join(ro_distdir, k)) self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): file_path = os.path.join(settings["DISTDIR"], k) self.assertTrue(os.path.islink(file_path)) with open(file_path, "rb") as f: self.assertEqual(f.read(), distfiles[k]) os.unlink(file_path) finally: settings.pop("PORTAGE_RO_DISTDIRS") settings["FETCHCOMMAND"] = orig_fetchcommand settings["RESUMECOMMAND"] = orig_resumecommand # Test local filesystem in GENTOO_MIRRORS orig_mirrors = settings["GENTOO_MIRRORS"] orig_fetchcommand = settings["FETCHCOMMAND"] try: settings["GENTOO_MIRRORS"] = ro_distdir settings["FETCHCOMMAND"] = settings["RESUMECOMMAND"] = "" self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) finally: settings["GENTOO_MIRRORS"] = orig_mirrors settings["FETCHCOMMAND"] = orig_fetchcommand settings["RESUMECOMMAND"] = orig_resumecommand # Test readonly DISTDIR orig_distdir_mode = os.stat(settings["DISTDIR"]).st_mode try: os.chmod(settings["DISTDIR"], 0o555) self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) finally: os.chmod(settings["DISTDIR"], orig_distdir_mode) # Test parallel-fetch mode settings["PORTAGE_PARALLEL_FETCHONLY"] = "1" try: self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) for k in settings["AA"].split(): os.unlink(os.path.join(settings["DISTDIR"], k)) self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) finally: settings.pop("PORTAGE_PARALLEL_FETCHONLY") # Test RESUMECOMMAND orig_resume_min_size = settings[ "PORTAGE_FETCH_RESUME_MIN_SIZE"] try: settings["PORTAGE_FETCH_RESUME_MIN_SIZE"] = "2" for k in settings["AA"].split(): file_path = os.path.join(settings["DISTDIR"], k) os.unlink(file_path) with open(file_path + _download_suffix, "wb") as f: f.write(distfiles[k][:2]) self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) finally: settings[ "PORTAGE_FETCH_RESUME_MIN_SIZE"] = orig_resume_min_size # Test readonly DISTDIR + skiprocheck, with FETCHCOMMAND set to temporarily chmod DISTDIR orig_fetchcommand = settings["FETCHCOMMAND"] orig_distdir_mode = os.stat(settings["DISTDIR"]).st_mode for k in settings["AA"].split(): os.unlink(os.path.join(settings["DISTDIR"], k)) try: os.chmod(settings["DISTDIR"], 0o555) settings["FETCHCOMMAND"] = ( '"%s" -c "chmod ug+w \\"${DISTDIR}\\"; %s; status=\\$?; chmod a-w \\"${DISTDIR}\\"; exit \\$status"' % (BASH_BINARY, orig_fetchcommand.replace('"', '\\"'))) settings.features.add("skiprocheck") settings.features.remove("distlocks") self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) finally: settings["FETCHCOMMAND"] = orig_fetchcommand os.chmod(settings["DISTDIR"], orig_distdir_mode) settings.features.remove("skiprocheck") settings.features.add("distlocks")