Ejemplo n.º 1
0
 def read(self, size):
     if self._eof:
         raise SubprocessStreamEOFError
     if self._pos + size <= self._buf_len:
         buf = bytes(self._buf[self._pos:self._pos + size])
         self._pos += size
         return buf
     else:
         size_left = size
         bio = BytesIO()
         while size_left:
             c = self._buf[self._pos:min(self._pos +
                                         size_left, self._buf_len)]
             bio.write(c)
             size_left -= len(c)
             self._pos += len(c)
             if size_left <= 0:
                 break
             self._q_req.put(options.cupid.mp_buffer_size)
             recv_size = self._q_rep.get()
             if recv_size < 0:
                 self._eof = True
                 break
             self._buf_len = recv_size
             self._pos = 0
         return bio.getvalue()
Ejemplo n.º 2
0
    def testBinaryImport(self):
        zip_io = StringIO()
        zip_f = zipfile.ZipFile(zip_io, 'w')
        zip_f.writestr('testa/a.so', '')
        zip_f.close()

        zip_io.seek(0)
        self.assertRaises(SystemError, CompressImporter,
                          zipfile.ZipFile(zip_io, 'r'))

        try:
            zip_io.seek(0)
            CompressImporter(zipfile.ZipFile(zip_io, 'r'),
                             extract=True,
                             _match_version=False)
            self.assertTrue(os.path.exists(CompressImporter._extract_path))
        finally:
            shutil.rmtree(CompressImporter._extract_path)
            CompressImporter._extract_path = None

        temp_path = tempfile.mkdtemp(prefix='tmp-pyodps-')
        lib_path = os.path.join(temp_path, 'mylib')
        os.makedirs(lib_path)

        lib_dict = dict()
        try:
            six_path = os.path.join(
                os.path.dirname(os.path.abspath(six.__file__)), 'six.py')
            shutil.copy(six_path, os.path.join(lib_path, 'five.py'))
            dummy_bin = open(os.path.join(lib_path, 'dummy.so'), 'w')
            dummy_bin.close()

            lib_files = ['five.py', 'dummy.so']
            lib_dict = dict((os.path.join(lib_path, fn),
                             open(os.path.join(lib_path, fn), 'r'))
                            for fn in lib_files)

            importer.ALLOW_BINARY = False
            self.assertRaises(SystemError, CompressImporter, lib_dict)

            importer.ALLOW_BINARY = True
            sys.meta_path.append(CompressImporter(lib_dict))
            import five
            self.assertEqual(list(to_binary('abc')),
                             list(five.binary_type(to_binary('abc'))))
        finally:
            [f.close() for f in six.itervalues(lib_dict)]
            shutil.rmtree(temp_path)
    def testRealImport(self):
        six_path = os.path.join(os.path.dirname(os.path.abspath(six.__file__)),
                                'six.py')
        zip_io = StringIO()
        zip_f = zipfile.ZipFile(zip_io, 'w')
        zip_f.write(six_path, arcname='mylib/five.py')
        zip_f.close()
        zip_io.seek(0)

        zip_f = zipfile.ZipFile(zip_io)

        sys.meta_path.append(CompressImporter(zip_f))

        import five
        self.assertEqual(list(to_binary('abc')),
                         list(five.binary_type(to_binary('abc'))))
Ejemplo n.º 4
0
 def read(self, size=-1):
     if Subprocess_StdIStream_eof(self._ptr):
         raise SubprocessStreamEOFError('Subprocess stream exhausted')
     if size < 0:
         buf_io = BytesIO()
         while True:
             try:
                 chunk = self.read(READ_CHUNK)
             except SubprocessStreamEOFError:
                 chunk = b''
             buf_io.write(chunk)
             if len(chunk) < READ_CHUNK:
                 break
         return buf_io.getvalue()
     else:
         p = create_string_buffer(size)
         Subprocess_StdIStream_read(self._ptr, p, size)
         return p.raw
Ejemplo n.º 5
0
    def testRealImport(self):
        raw_meta_path = sys.meta_path

        six_path = os.path.join(os.path.dirname(os.path.abspath(six.__file__)), 'six.py')
        zip_io = StringIO()
        zip_f = zipfile.ZipFile(zip_io, 'w')
        zip_f.write(six_path, arcname='mylib/five.py')
        zip_f.close()
        zip_io.seek(0)

        zip_f = zipfile.ZipFile(zip_io)

        sys.meta_path.append(CompressImporter(zip_f))

        try:
            import five
            self.assertEqual(list(to_binary('abc')), list(five.binary_type(to_binary('abc'))))
        finally:
            sys.meta_path = raw_meta_path
    def testImport(self):
        raw_meta_path = sys.meta_path

        zip_io = StringIO()
        zip_f = zipfile.ZipFile(zip_io, 'w')
        zip_f.writestr('testa/a.py', 'a = 1')
        zip_f.close()

        tar_io = StringIO()
        tar_f = tarfile.TarFile(fileobj=tar_io, mode='w')
        tar_f.addfile(tarfile.TarInfo(name='testb/__init__.py'),
                      fileobj=StringIO())
        info = tarfile.TarInfo(name='testb/b.py')
        c = to_binary('from a import a; b = a + 1')
        s = StringIO(c)
        info.size = len(c)
        tar_f.addfile(info, fileobj=s)
        tar_f.close()

        zip_io.seek(0)
        tar_io.seek(0)

        zip_f = zipfile.ZipFile(zip_io)
        tar_f = tarfile.TarFile(fileobj=tar_io)

        sys.meta_path.append(CompressImporter(zip_f, tar_f))

        try:
            from testb.b import b
            self.assertEqual(b, 2)
        finally:
            sys.meta_path = raw_meta_path
Ejemplo n.º 7
0
    def testImport(self):
        raw_meta_path = sys.meta_path

        zip_io = StringIO()
        zip_f = zipfile.ZipFile(zip_io, 'w')
        zip_f.writestr('testa/a.py', 'a = 1')
        zip_f.close()

        tar_io = StringIO()
        tar_f = tarfile.TarFile(fileobj=tar_io, mode='w')
        tar_f.addfile(tarfile.TarInfo(name='testb/__init__.py'), fileobj=StringIO())
        info = tarfile.TarInfo(name='testb/b.py')
        c = to_binary('from a import a; b = a + 1')
        s = StringIO(c)
        info.size = len(c)
        tar_f.addfile(info, fileobj=s)
        tar_f.close()

        zip_io.seek(0)
        tar_io.seek(0)

        zip_f = zipfile.ZipFile(zip_io)
        tar_f = tarfile.TarFile(fileobj=tar_io)

        sys.meta_path.append(CompressImporter(zip_f, tar_f))

        try:
            from testb.b import b
            self.assertEqual(b, 2)
        finally:
            sys.meta_path = raw_meta_path
    def testImport(self):
        zip_io = StringIO()
        zip_f = zipfile.ZipFile(zip_io, 'w')
        zip_f.writestr('testa/a.py', 'a = 1')
        zip_f.close()

        tar_io = StringIO()
        tar_f = tarfile.TarFile(fileobj=tar_io, mode='w')
        tar_f.addfile(tarfile.TarInfo(name='testb/__init__.py'),
                      fileobj=StringIO())
        info = tarfile.TarInfo(name='testb/b.py')
        c = to_binary('from a import a; b = a + 1')
        s = StringIO(c)
        info.size = len(c)
        tar_f.addfile(info, fileobj=s)
        tar_f.close()

        dict_io_init = dict()
        dict_io_init['/opt/test_pyodps_dev/testc/__init__.py'] = StringIO()
        dict_io_init['/opt/test_pyodps_dev/testc/c.py'] = StringIO(
            to_binary('from a import a; c = a + 2'))

        dict_io_file = dict()
        dict_io_file['/opt/test_pyodps_dev/testd/d.py'] = StringIO(
            to_binary('from a import a; d = a + 3'))

        zip_io.seek(0)
        tar_io.seek(0)

        zip_f = zipfile.ZipFile(zip_io)
        tar_f = tarfile.TarFile(fileobj=tar_io)

        importer.ALLOW_BINARY = False
        imp = CompressImporter(zip_f, tar_f, dict_io_init, dict_io_file)
        self.assertEqual(len(imp._files), 4)
        sys.meta_path.append(imp)

        from testb.b import b
        self.assertEqual(b, 2)
        from testc.c import c
        self.assertEqual(c, 3)
        self.assertRaises(ImportError, __import__, 'c', fromlist=[])
        from d import d
        self.assertEqual(d, 4)
Ejemplo n.º 9
0
    def testBinaryImport(self):
        zip_io = StringIO()
        zip_f = zipfile.ZipFile(zip_io, 'w')
        zip_f.writestr('testa/a.so', '')
        zip_f.close()

        zip_io.seek(0)
        self.assertRaises(SystemError, CompressImporter,
                          zipfile.ZipFile(zip_io, 'r'))

        try:
            zip_io.seek(0)
            CompressImporter(zipfile.ZipFile(zip_io, 'r'),
                             extract=True,
                             _match_version=False)
            self.assertTrue(os.path.exists(CompressImporter._extract_path))
        finally:
            shutil.rmtree(CompressImporter._extract_path)
            CompressImporter._extract_path = None

        six_path = os.path.join(os.path.dirname(os.path.abspath(six.__file__)),
                                'six.py')

        temp_path = tempfile.mkdtemp(prefix='tmp-pyodps-')
        lib_path = os.path.join(temp_path, 'mylib')
        lib_path2 = os.path.join(temp_path, 'mylib2')
        os.makedirs(lib_path)
        os.makedirs(lib_path2)

        lib_dict = dict()
        try:
            with open(os.path.join(lib_path, '__init__.py'), 'w'):
                pass
            shutil.copy(six_path, os.path.join(lib_path, 'fake_six.py'))
            dummy_bin = open(os.path.join(lib_path, 'dummy.so'), 'w')
            dummy_bin.close()

            sub_lib_path = os.path.join(lib_path, 'sub_path')
            os.makedirs(sub_lib_path)

            with open(os.path.join(sub_lib_path, '__init__.py'), 'w'):
                pass
            shutil.copy(six_path, os.path.join(sub_lib_path, 'fake_six.py'))

            lib_files = [
                '__init__.py', 'fake_six.py', 'dummy.so',
                os.path.join('sub_path', '__init__.py'),
                os.path.join('sub_path', 'fake_six.py')
            ]
            lib_dict = dict((os.path.join(lib_path, fn),
                             open(os.path.join(lib_path, fn), 'r'))
                            for fn in lib_files)

            importer.ALLOW_BINARY = False
            self.assertRaises(SystemError, CompressImporter, lib_dict)

            importer.ALLOW_BINARY = True
            importer_obj = CompressImporter(lib_dict)
            sys.meta_path.append(importer_obj)
            from mylib import fake_six
            self.assertEqual(list(to_binary('abc')),
                             list(fake_six.binary_type(to_binary('abc'))))
            self.assertRaises(ImportError, __import__, 'sub_path', fromlist=[])

            with open(os.path.join(lib_path2, '__init__.py'), 'w'):
                pass
            shutil.copy(six_path, os.path.join(lib_path2, 'fake_six.py'))
            dummy_bin = open(os.path.join(lib_path2, 'dummy.so'), 'w')
            dummy_bin.close()

            sub_lib_path = os.path.join(lib_path2, 'sub_path2')
            os.makedirs(sub_lib_path)

            with open(os.path.join(sub_lib_path, '__init__.py'), 'w'):
                pass
            shutil.copy(six_path, os.path.join(sub_lib_path, 'fake_six.py'))

            lib_files = [
                '__init__.py', 'fake_six.py', 'dummy.so',
                os.path.join('sub_path2', '__init__.py'),
                os.path.join('sub_path2', 'fake_six.py')
            ]

            importer.ALLOW_BINARY = True
            importer_obj = CompressImporter(lib_files)
            sys.meta_path.append(importer_obj)
            from mylib2 import fake_six
            self.assertEqual(list(to_binary('abc')),
                             list(fake_six.binary_type(to_binary('abc'))))
            self.assertRaises(ImportError,
                              __import__,
                              'sub_path2',
                              fromlist=[])
        finally:
            [f.close() for f in six.itervalues(lib_dict)]
            shutil.rmtree(temp_path)