Ejemplo n.º 1
0
    def _testPipeReader(self, master_fd, slave_fd, test_string):
        """
		Use a poll loop to read data from a pipe and assert that
		the data written to the pipe is identical to the data
		read from the pipe.
		"""

        # WARNING: It is very important to use unbuffered mode here,
        # in order to avoid issue 5380 with python3.
        master_file = os.fdopen(master_fd, 'rb', 0)
        scheduler = global_event_loop()

        consumer = PipeReader(input_files={"producer": master_file},
                              _use_array=self._use_array,
                              scheduler=scheduler)
        consumer.start()

        producer = scheduler.run_until_complete(
            asyncio.create_subprocess_exec("bash",
                                           "-c",
                                           self._echo_cmd % test_string,
                                           stdout=slave_fd,
                                           loop=scheduler))

        os.close(slave_fd)
        scheduler.run_until_complete(producer.wait())
        scheduler.run_until_complete(consumer.async_wait())

        self.assertEqual(producer.returncode, os.EX_OK)
        self.assertEqual(consumer.returncode, os.EX_OK)

        return consumer.getvalue().decode('ascii', 'replace')
Ejemplo n.º 2
0
	def _testPipeReader(self, master_fd, slave_fd, test_string):
		"""
		Use a poll loop to read data from a pipe and assert that
		the data written to the pipe is identical to the data
		read from the pipe.
		"""

		# WARNING: It is very important to use unbuffered mode here,
		# in order to avoid issue 5380 with python3.
		master_file = os.fdopen(master_fd, 'rb', 0)
		scheduler = global_event_loop()

		consumer = PipeReader(
			input_files={"producer" : master_file},
			_use_array=self._use_array,
			scheduler=scheduler)

		producer = PopenProcess(
			pipe_reader=consumer,
			proc=subprocess.Popen(["bash", "-c", self._echo_cmd % test_string],
				stdout=slave_fd),
			scheduler=scheduler)

		producer.start()
		os.close(slave_fd)
		producer.wait()
		consumer.wait()

		self.assertEqual(producer.returncode, os.EX_OK)
		self.assertEqual(consumer.returncode, os.EX_OK)

		return consumer.getvalue().decode('ascii', 'replace')
Ejemplo n.º 3
0
    def _receive_reply(self, input_fd):

        start_time = time.time()

        pipe_reader = PipeReader(input_files={"input_fd": input_fd},
                                 scheduler=global_event_loop())
        pipe_reader.start()

        eof = pipe_reader.poll() is not None

        while not eof:
            pipe_reader._wait_loop(timeout=self._COMMUNICATE_RETRY_TIMEOUT)
            eof = pipe_reader.poll() is not None
            if not eof:
                if self._daemon_is_alive():
                    self._timeout_retry_msg(
                        start_time, portage.localization._("during read"))
                else:
                    pipe_reader.cancel()
                    self._no_daemon_msg()
                    return 2

        buf = pipe_reader.getvalue()

        retval = 2

        if not buf:

            portage.util.writemsg_level(
                "ebuild-ipc: %s\n" % (portage.localization._("read failed"), ),
                level=logging.ERROR,
                noiselevel=-1,
            )

        else:

            try:
                reply = pickle.loads(buf)
            except SystemExit:
                raise
            except Exception as e:
                # The pickle module can raise practically
                # any exception when given corrupt data.
                portage.util.writemsg_level("ebuild-ipc: %s\n" % (e, ),
                                            level=logging.ERROR,
                                            noiselevel=-1)

            else:

                (out, err, retval) = reply

                if out:
                    portage.util.writemsg_stdout(out, noiselevel=-1)

                if err:
                    portage.util.writemsg(err, noiselevel=-1)

        return retval
Ejemplo n.º 4
0
	def _receive_reply(self, input_fd):

		start_time = time.time()

		pipe_reader = PipeReader(input_files={"input_fd":input_fd},
			scheduler=global_event_loop())
		pipe_reader.start()

		eof = pipe_reader.poll() is not None

		while not eof:
			pipe_reader._wait_loop(timeout=self._COMMUNICATE_RETRY_TIMEOUT)
			eof = pipe_reader.poll() is not None
			if not eof:
				if self._daemon_is_alive():
					self._timeout_retry_msg(start_time,
						portage.localization._('during read'))
				else:
					pipe_reader.cancel()
					self._no_daemon_msg()
					return 2

		buf = pipe_reader.getvalue()

		retval = 2

		if not buf:

			portage.util.writemsg_level(
				"ebuild-ipc: %s\n" % \
				(portage.localization._('read failed'),),
				level=logging.ERROR, noiselevel=-1)

		else:

			try:
				reply = pickle.loads(buf)
			except SystemExit:
				raise
			except Exception as e:
				# The pickle module can raise practically
				# any exception when given corrupt data.
				portage.util.writemsg_level(
					"ebuild-ipc: %s\n" % (e,),
					level=logging.ERROR, noiselevel=-1)

			else:

				(out, err, retval) = reply

				if out:
					portage.util.writemsg_stdout(out, noiselevel=-1)

				if err:
					portage.util.writemsg(err, noiselevel=-1)

		return retval
    def testLazyImportPortageBaseline(self):
        """
        Check what modules are imported by a baseline module import.
        """

        env = os.environ.copy()
        pythonpath = env.get("PYTHONPATH")
        if pythonpath is not None and not pythonpath.strip():
            pythonpath = None
        if pythonpath is None:
            pythonpath = ""
        else:
            pythonpath = ":" + pythonpath
        pythonpath = PORTAGE_PYM_PATH + pythonpath
        env["PYTHONPATH"] = pythonpath

        # If python is patched to insert the path of the
        # currently installed portage module into sys.path,
        # then the above PYTHONPATH override doesn't help.
        env["PORTAGE_PYM_PATH"] = PORTAGE_PYM_PATH

        scheduler = global_event_loop()
        master_fd, slave_fd = os.pipe()
        master_file = os.fdopen(master_fd, "rb", 0)
        slave_file = os.fdopen(slave_fd, "wb")
        producer = SpawnProcess(
            args=self._baseline_import_cmd,
            env=env,
            fd_pipes={1: slave_fd},
            scheduler=scheduler,
        )
        producer.start()
        slave_file.close()

        consumer = PipeReader(input_files={"producer": master_file},
                              scheduler=scheduler)

        consumer.start()
        consumer.wait()
        self.assertEqual(producer.wait(), os.EX_OK)
        self.assertEqual(consumer.wait(), os.EX_OK)

        output = consumer.getvalue().decode("ascii", "replace").split()

        unexpected_modules = " ".join(
            sorted(x for x in output if self._module_re.match(x) is not None
                   and x not in self._baseline_imports))

        self.assertEqual("", unexpected_modules)
	def testLazyImportPortageBaseline(self):
		"""
		Check what modules are imported by a baseline module import.
		"""

		env = os.environ.copy()
		pythonpath = env.get('PYTHONPATH')
		if pythonpath is not None and not pythonpath.strip():
			pythonpath = None
		if pythonpath is None:
			pythonpath = ''
		else:
			pythonpath = ':' + pythonpath
		pythonpath = PORTAGE_PYM_PATH + pythonpath
		env['PYTHONPATH'] = pythonpath

		# If python is patched to insert the path of the
		# currently installed portage module into sys.path,
		# then the above PYTHONPATH override doesn't help.
		env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH

		scheduler = PollScheduler().sched_iface
		master_fd, slave_fd = os.pipe()
		master_file = os.fdopen(master_fd, 'rb', 0)
		slave_file = os.fdopen(slave_fd, 'wb')
		producer = SpawnProcess(
			args=self._baseline_import_cmd,
			env=env, fd_pipes={1:slave_fd},
			scheduler=scheduler)
		producer.start()
		slave_file.close()

		consumer = PipeReader(
			input_files={"producer" : master_file},
			scheduler=scheduler)

		consumer.start()
		consumer.wait()
		self.assertEqual(producer.wait(), os.EX_OK)
		self.assertEqual(consumer.wait(), os.EX_OK)

		output = consumer.getvalue().decode('ascii', 'replace').split()

		unexpected_modules = " ".join(sorted(x for x in output \
			if self._module_re.match(x) is not None and \
			x not in self._baseline_imports))

		self.assertEqual("", unexpected_modules)
Ejemplo n.º 7
0
    def _testPipeReader(self, test_string):
        """
		Use a poll loop to read data from a pipe and assert that
		the data written to the pipe is identical to the data
		read from the pipe.
		"""

        if self._use_pty:
            got_pty, master_fd, slave_fd = _create_pty_or_pipe()
            if not got_pty:
                os.close(slave_fd)
                os.close(master_fd)
                skip_reason = "pty not acquired"
                self.portage_skip = skip_reason
                self.fail(skip_reason)
                return
        else:
            master_fd, slave_fd = os.pipe()

        # WARNING: It is very important to use unbuffered mode here,
        # in order to avoid issue 5380 with python3.
        master_file = os.fdopen(master_fd, 'rb', 0)
        slave_file = os.fdopen(slave_fd, 'wb', 0)
        task_scheduler = TaskScheduler(max_jobs=2)
        producer = SpawnProcess(
            args=["bash", "-c", self._echo_cmd % test_string],
            env=os.environ,
            fd_pipes={1: slave_fd},
            scheduler=task_scheduler.sched_iface)
        task_scheduler.add(producer)
        slave_file.close()

        consumer = PipeReader(input_files={"producer": master_file},
                              scheduler=task_scheduler.sched_iface,
                              _use_array=self._use_array)

        task_scheduler.add(consumer)

        # This will ensure that both tasks have exited, which
        # is necessary to avoid "ResourceWarning: unclosed file"
        # warnings since Python 3.2 (and also ensures that we
        # don't leave any zombie child processes).
        task_scheduler.run()
        self.assertEqual(producer.returncode, os.EX_OK)
        self.assertEqual(consumer.returncode, os.EX_OK)

        return consumer.getvalue().decode('ascii', 'replace')
Ejemplo n.º 8
0
	def _testPipeReader(self, test_string):
		"""
		Use a poll loop to read data from a pipe and assert that
		the data written to the pipe is identical to the data
		read from the pipe.
		"""

		if self._use_pty:
			got_pty, master_fd, slave_fd = _create_pty_or_pipe()
			if not got_pty:
				os.close(slave_fd)
				os.close(master_fd)
				skip_reason = "pty not acquired"
				self.portage_skip = skip_reason
				self.fail(skip_reason)
				return
		else:
			master_fd, slave_fd = os.pipe()

		# WARNING: It is very important to use unbuffered mode here,
		# in order to avoid issue 5380 with python3.
		master_file = os.fdopen(master_fd, 'rb', 0)
		slave_file = os.fdopen(slave_fd, 'wb', 0)
		task_scheduler = TaskScheduler(max_jobs=2)
		producer = SpawnProcess(
			args=["bash", "-c", self._echo_cmd % test_string],
			env=os.environ, fd_pipes={1:slave_fd},
			scheduler=task_scheduler.sched_iface)
		task_scheduler.add(producer)
		slave_file.close()

		consumer = PipeReader(
			input_files={"producer" : master_file},
			scheduler=task_scheduler.sched_iface, _use_array=self._use_array)

		task_scheduler.add(consumer)

		# This will ensure that both tasks have exited, which
		# is necessary to avoid "ResourceWarning: unclosed file"
		# warnings since Python 3.2 (and also ensures that we
		# don't leave any zombie child processes).
		task_scheduler.run()
		self.assertEqual(producer.returncode, os.EX_OK)
		self.assertEqual(consumer.returncode, os.EX_OK)

		return consumer.getvalue().decode('ascii', 'replace')
Ejemplo n.º 9
0
	def _testPipeReader(self, test_string):
		"""
		Use a poll loop to read data from a pipe and assert that
		the data written to the pipe is identical to the data
		read from the pipe.
		"""

		if self._use_pty:
			got_pty, master_fd, slave_fd = _create_pty_or_pipe()
			if not got_pty:
				os.close(slave_fd)
				os.close(master_fd)
				skip_reason = "pty not acquired"
				self.portage_skip = skip_reason
				self.fail(skip_reason)
				return
		else:
			master_fd, slave_fd = os.pipe()

		# WARNING: It is very important to use unbuffered mode here,
		# in order to avoid issue 5380 with python3.
		master_file = os.fdopen(master_fd, 'rb', 0)
		scheduler = global_event_loop()

		consumer = PipeReader(
			input_files={"producer" : master_file},
			_use_array=self._use_array,
			scheduler=scheduler)

		producer = PopenProcess(
			pipe_reader=consumer,
			proc=subprocess.Popen(["bash", "-c", self._echo_cmd % test_string],
				stdout=slave_fd),
			scheduler=scheduler)

		producer.start()
		os.close(slave_fd)
		producer.wait()

		self.assertEqual(producer.returncode, os.EX_OK)
		self.assertEqual(consumer.returncode, os.EX_OK)

		return consumer.getvalue().decode('ascii', 'replace')
Ejemplo n.º 10
0
	def testPipeReader(self):
		"""
		Use a poll loop to read data from a pipe and assert that
		the data written to the pipe is identical to the data
		read from the pipe.
		"""

		test_string = 2 * "blah blah blah\n"

		master_fd, slave_fd = self._create_pipe()
		master_file = os.fdopen(master_fd, 'rb')

		task_scheduler = TaskScheduler(max_jobs=2)
		scheduler = task_scheduler.sched_iface

		class Producer(SpawnProcess):
			def _spawn(self, args, **kwargs):
				rval = SpawnProcess._spawn(self, args, **kwargs)
				os.close(kwargs['fd_pipes'][1])
				return rval

		producer = Producer(
			args=["bash", "-c", "echo -n '%s'" % test_string],
			fd_pipes={1:slave_fd}, scheduler=scheduler)

		consumer = PipeReader(
			input_files={"producer" : master_file},
			scheduler=scheduler)

		task_scheduler.add(producer)
		task_scheduler.add(consumer)

		task_scheduler.run()

		if sys.hexversion >= 0x3000000:
			test_string = test_string.encode()

		self._assertEqual(test_string, consumer.getvalue())
Ejemplo n.º 11
0
	def testPipeReader(self):
		"""
		Use a poll loop to read data from a pipe and assert that
		the data written to the pipe is identical to the data
		read from the pipe.
		"""

		test_string = 2 * "blah blah blah\n"

		scheduler = PollScheduler().sched_iface
		master_fd, slave_fd = os.pipe()
		master_file = os.fdopen(master_fd, 'rb', 0)
		slave_file = os.fdopen(slave_fd, 'wb')
		producer = SpawnProcess(
			args=["bash", "-c", "echo -n '%s'" % test_string],
			env=os.environ, fd_pipes={1:slave_fd},
			scheduler=scheduler)
		producer.start()
		slave_file.close()

		consumer = PipeReader(
			input_files={"producer" : master_file},
			scheduler=scheduler)

		consumer.start()

		# This will ensure that both tasks have exited, which
		# is necessary to avoid "ResourceWarning: unclosed file"
		# warnings since Python 3.2 (and also ensures that we
		# don't leave any zombie child processes).
		scheduler.schedule()
		self.assertEqual(producer.returncode, os.EX_OK)
		self.assertEqual(consumer.returncode, os.EX_OK)

		output = consumer.getvalue().decode('ascii', 'replace')
		self.assertEqual(test_string, output)
Ejemplo n.º 12
0
    def testDoebuild(self):
        """
        Invoke portage.doebuild() with the fd_pipes parameter, and
        check that the expected output appears in the pipe. This
        functionality is not used by portage internally, but it is
        supported for API consumers (see bug #475812).
        """

        output_fd = 200
        ebuild_body = ["S=${WORKDIR}"]
        for phase_func in (
                "pkg_info",
                "pkg_nofetch",
                "pkg_pretend",
                "pkg_setup",
                "src_unpack",
                "src_prepare",
                "src_configure",
                "src_compile",
                "src_test",
                "src_install",
        ):
            ebuild_body.append(("%s() { echo ${EBUILD_PHASE}"
                                " 1>&%s; }") % (phase_func, output_fd))

        ebuild_body.append("")
        ebuild_body = "\n".join(ebuild_body)

        ebuilds = {
            "app-misct/foo-1": {
                "EAPI": "5",
                "MISC_CONTENT": ebuild_body,
            }
        }

        # Override things that may be unavailable, or may have portability
        # issues when running tests in exotic environments.
        #   prepstrip - bug #447810 (bash read builtin EINTR problem)
        true_symlinks = ("find", "prepstrip", "sed", "scanelf")
        true_binary = portage.process.find_binary("true")
        self.assertEqual(true_binary is None, False, "true command not found")

        dev_null = open(os.devnull, "wb")
        playground = ResolverPlayground(ebuilds=ebuilds)
        try:
            QueryCommand._db = playground.trees
            root_config = playground.trees[playground.eroot]["root_config"]
            portdb = root_config.trees["porttree"].dbapi
            settings = portage.config(clone=playground.settings)
            if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
                settings["__PORTAGE_TEST_HARDLINK_LOCKS"] = os.environ[
                    "__PORTAGE_TEST_HARDLINK_LOCKS"]
                settings.backup_changes("__PORTAGE_TEST_HARDLINK_LOCKS")

            settings.features.add("noauto")
            settings.features.add("test")
            settings["PORTAGE_PYTHON"] = portage._python_interpreter
            settings["PORTAGE_QUIET"] = "1"
            settings["PYTHONDONTWRITEBYTECODE"] = os.environ.get(
                "PYTHONDONTWRITEBYTECODE", "")

            fake_bin = os.path.join(settings["EPREFIX"], "bin")
            portage.util.ensure_dirs(fake_bin)
            for x in true_symlinks:
                os.symlink(true_binary, os.path.join(fake_bin, x))

            settings["__PORTAGE_TEST_PATH_OVERRIDE"] = fake_bin
            settings.backup_changes("__PORTAGE_TEST_PATH_OVERRIDE")

            cpv = "app-misct/foo-1"
            metadata = dict(
                zip(Package.metadata_keys,
                    portdb.aux_get(cpv, Package.metadata_keys)))

            pkg = Package(
                built=False,
                cpv=cpv,
                installed=False,
                metadata=metadata,
                root_config=root_config,
                type_name="ebuild",
            )
            settings.setcpv(pkg)
            ebuildpath = portdb.findname(cpv)
            self.assertNotEqual(ebuildpath, None)

            for phase in (
                    "info",
                    "nofetch",
                    "pretend",
                    "setup",
                    "unpack",
                    "prepare",
                    "configure",
                    "compile",
                    "test",
                    "install",
                    "qmerge",
                    "clean",
                    "merge",
            ):

                pr, pw = os.pipe()

                producer = DoebuildProcess(
                    doebuild_pargs=(ebuildpath, phase),
                    doebuild_kwargs={
                        "settings": settings,
                        "mydbapi": portdb,
                        "tree": "porttree",
                        "vartree": root_config.trees["vartree"],
                        "fd_pipes": {
                            1: dev_null.fileno(),
                            2: dev_null.fileno(),
                            output_fd: pw,
                        },
                        "prev_mtimes": {},
                    },
                )

                consumer = PipeReader(input_files={"producer": pr})

                task_scheduler = TaskScheduler(iter([producer, consumer]),
                                               max_jobs=2)

                try:
                    task_scheduler.start()
                finally:
                    # PipeReader closes pr
                    os.close(pw)

                task_scheduler.wait()
                output = portage._unicode_decode(
                    consumer.getvalue()).rstrip("\n")

                if task_scheduler.returncode != os.EX_OK:
                    portage.writemsg(output, noiselevel=-1)

                self.assertEqual(task_scheduler.returncode, os.EX_OK)

                if phase not in ("clean", "merge", "qmerge"):
                    self.assertEqual(phase, output)

        finally:
            dev_null.close()
            playground.cleanup()
            QueryCommand._db = None
Ejemplo n.º 13
0
	def testDoebuild(self):
		"""
		Invoke portage.doebuild() with the fd_pipes parameter, and
		check that the expected output appears in the pipe. This
		functionality is not used by portage internally, but it is
		supported for API consumers (see bug #475812).
		"""

		output_fd = 200
		ebuild_body = ['S=${WORKDIR}']
		for phase_func in ('pkg_info', 'pkg_nofetch', 'pkg_pretend',
			'pkg_setup', 'src_unpack', 'src_prepare', 'src_configure',
			'src_compile', 'src_test', 'src_install'):
			ebuild_body.append(('%s() { echo ${EBUILD_PHASE}'
				' 1>&%s; }') % (phase_func, output_fd))

		ebuild_body.append('')
		ebuild_body = '\n'.join(ebuild_body)

		ebuilds = {
			'app-misct/foo-1': {
				'EAPI'      : '5',
				"MISC_CONTENT": ebuild_body,
			}
		}

		# Override things that may be unavailable, or may have portability
		# issues when running tests in exotic environments.
		#   prepstrip - bug #447810 (bash read builtin EINTR problem)
		true_symlinks = ("find", "prepstrip", "sed", "scanelf")
		true_binary = portage.process.find_binary("true")
		self.assertEqual(true_binary is None, False,
			"true command not found")

		dev_null = open(os.devnull, 'wb')
		playground = ResolverPlayground(ebuilds=ebuilds)
		try:
			QueryCommand._db = playground.trees
			root_config = playground.trees[playground.eroot]['root_config']
			portdb = root_config.trees["porttree"].dbapi
			settings = portage.config(clone=playground.settings)
			if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
				settings["__PORTAGE_TEST_HARDLINK_LOCKS"] = \
					os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"]
				settings.backup_changes("__PORTAGE_TEST_HARDLINK_LOCKS")

			settings.features.add("noauto")
			settings.features.add("test")
			settings['PORTAGE_PYTHON'] = portage._python_interpreter
			settings['PORTAGE_QUIET'] = "1"
			settings['PYTHONDONTWRITEBYTECODE'] = os.environ.get("PYTHONDONTWRITEBYTECODE", "")

			fake_bin = os.path.join(settings["EPREFIX"], "bin")
			portage.util.ensure_dirs(fake_bin)
			for x in true_symlinks:
				os.symlink(true_binary, os.path.join(fake_bin, x))

			settings["__PORTAGE_TEST_PATH_OVERRIDE"] = fake_bin
			settings.backup_changes("__PORTAGE_TEST_PATH_OVERRIDE")

			cpv = 'app-misct/foo-1'
			metadata = dict(zip(Package.metadata_keys,
				portdb.aux_get(cpv, Package.metadata_keys)))

			pkg = Package(built=False, cpv=cpv, installed=False,
				metadata=metadata, root_config=root_config,
				type_name='ebuild')
			settings.setcpv(pkg)
			ebuildpath = portdb.findname(cpv)
			self.assertNotEqual(ebuildpath, None)

			for phase in ('info', 'nofetch',
				 'pretend', 'setup', 'unpack', 'prepare', 'configure',
				 'compile', 'test', 'install', 'qmerge', 'clean', 'merge'):

				pr, pw = os.pipe()

				producer = DoebuildProcess(doebuild_pargs=(ebuildpath, phase),
					doebuild_kwargs={"settings" : settings,
						"mydbapi": portdb, "tree": "porttree",
						"vartree": root_config.trees["vartree"],
						"fd_pipes": {
							1: dev_null.fileno(),
							2: dev_null.fileno(),
							output_fd: pw,
						},
						"prev_mtimes": {}})

				consumer = PipeReader(
					input_files={"producer" : pr})

				task_scheduler = TaskScheduler(iter([producer, consumer]),
					max_jobs=2)

				try:
					task_scheduler.start()
				finally:
					# PipeReader closes pr
					os.close(pw)

				task_scheduler.wait()
				output = portage._unicode_decode(
					consumer.getvalue()).rstrip("\n")

				if task_scheduler.returncode != os.EX_OK:
					portage.writemsg(output, noiselevel=-1)

				self.assertEqual(task_scheduler.returncode, os.EX_OK)

				if phase not in ('clean', 'merge', 'qmerge'):
					self.assertEqual(phase, output)

		finally:
			dev_null.close()
			playground.cleanup()
			QueryCommand._db = None