예제 #1
0
def is_stdin(file: IO) -> bool:
    try:
        file_no = file.fileno()
    except Exception:
        return False
    else:
        return file_no == sys.stdin.fileno()
예제 #2
0
    def upload_file(self, file_handle: IO, file_url: str) -> None:
        assert self.project_details

        content_type = magic.from_descriptor(file_handle.fileno(), mime=True)

        headers = {
            "x-amz-acl":
            "private" if self.project_details["private"] else "public-read",
            "Content-Type": content_type,
        }

        # This is weird but requests handles an empty file object
        # differently than None, with empty file
        # it appends Content-Length: 0 header which triggers 501 on S3
        if not file_handle.read(1):
            data = None
        else:
            data = file_handle
            file_handle.seek(0)

        try:
            response = requests.put(file_url, data=data, headers=headers)
            response.raise_for_status()
        except HTTPError as e:
            raise FileUploadFailed(
                f"There was an error uploading the file. Error code: {response.status_code}"
            ) from e

        file_name = os.path.basename(file_handle.name)
        print(f"File {file_name} uploaded to s3")
예제 #3
0
 def __init__(self, f: IO):
     self.attributes = {}
     for attribute in ("seekable", "readable", "writeable"):
         if not hasattr(f, attribute):
             continue
         self.attributes[attribute] = getattr(f, attribute)()
     self.mmap = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
예제 #4
0
 def print_file(
     self, fileobj: IO, fmt: str = "tree", path: Optional[str] = None
 ) -> None:
     fmt = schema_out_format(fmt)
     ret = lib.lys_print_fd(fileobj.fileno(), self.cdata, fmt, str2c(path), 0, 0)
     if ret != 0:
         raise self.context.error("cannot print module")
예제 #5
0
    def parse_module_file(self, fileobj: IO, fmt: str = "yang") -> Module:
        if self.cdata is None:
            raise RuntimeError("context already destroyed")
        fmt = schema_in_format(fmt)
        mod = lib.lys_parse_fd(self.cdata, fileobj.fileno(), fmt)
        if not mod:
            raise self.error("cannot parse module")

        return Module(self, mod)
예제 #6
0
    def unlock(file_: typing.IO):
        try:
            savepos = file_.tell()
            if savepos:
                file_.seek(0)

            try:
                msvcrt.locking(file_.fileno(), constants.LockFlags.UNBLOCK,
                               lock_length)
            except IOError as exc:
                exception = exc
                if exc.strerror == 'Permission denied':
                    hfile = win32file._get_osfhandle(file_.fileno())
                    try:
                        win32file.UnlockFileEx(hfile, 0, -0x10000,
                                               __overlapped)
                    except pywintypes.error as exc:
                        exception = exc
                        if exc.winerror == winerror.ERROR_NOT_LOCKED:
                            # error: (158, 'UnlockFileEx',
                            #         'The segment is already unlocked.')
                            # To match the 'posix' implementation, silently
                            # ignore this error
                            pass
                        else:
                            # Q:  Are there exceptions/codes we should be
                            # dealing with here?
                            raise
                else:
                    raise exceptions.LockException(
                        exceptions.LockException.LOCK_FAILED,
                        exception.strerror,
                        fh=file_)
            finally:
                if savepos:
                    file_.seek(savepos)
        except IOError as exc:
            raise exceptions.LockException(
                exceptions.LockException.LOCK_FAILED, exc.strerror, fh=file_)
예제 #7
0
    def lock(file_: typing.IO, flags: constants.LockFlags):
        locking_exceptions = IOError,
        try:  # pragma: no cover
            locking_exceptions += BlockingIOError,  # type: ignore
        except NameError:  # pragma: no cover
            pass

        try:
            fcntl.flock(file_.fileno(), flags)
        except locking_exceptions as exc_value:
            # The exception code varies on different systems so we'll catch
            # every IO error
            raise exceptions.LockException(exc_value, fh=file_)
예제 #8
0
def flush_file_handle(file_handle: IO):
    """
    Attempts to flush the handle. First tries to call .flush and then
    tries to apply `os.fsync` to the file descriptor to truly flush the
    buffer to the file.

    :param file_handle: The file handle to attempt to flush
    :type file_handle: IO
    """
    if hasattr(file_handle, 'flush'):
        file_handle.flush()
    if hasattr(file_handle, 'fileno'):
        os.fsync(file_handle.fileno())
예제 #9
0
def get_file_name(file: t.IO) -> str:
    """Get the path of `file`."""
    handle = msvcrt.get_osfhandle(file.fileno())
    buffer = ctypes.create_unicode_buffer(wintypes.MAX_PATH)

    ret_val = GetFinalPathNameByHandle(handle, buffer, wintypes.MAX_PATH, 0)
    if ret_val > wintypes.MAX_PATH:
        buffer = ctypes.create_unicode_buffer(ret_val)
        ret_val = GetFinalPathNameByHandle(handle, buffer, ret_val, 0)

    if ret_val == 0:
        raise ctypes.WinError()

    return buffer.value.removeprefix("\\\\?\\")
예제 #10
0
def silence(fd: IO):
    """Silence any output from fd."""
    from os import close, dup, dup2, fdopen, pipe

    # Do not silence when debugging.
    if not DEBUG:
        # Backup the file
        old_fd = fd

        # Flush the file so it can be silenced properly.
        fd.flush()

        # Create a duplicate of fd
        new_fd = dup(fd.fileno())

        # Create a pipe to write to.
        read, write = pipe()

        # Set the write to the fd filenumber
        dup2(write, fd.fileno())

        # Close the pipe.
        close(write)
        close(read)

        # Set fd to the new fd
        fd = fdopen(new_fd, 'w')

    try:
        # Run the commands in the 'with' statement.
        yield
    finally:
        if not DEBUG:
            # Return the fd back to its original state.
            dup2(fd.fileno(), old_fd.fileno())
            fd = old_fd
예제 #11
0
 def parse_data_file(
     self,
     fileobj: IO,
     fmt: str,
     data: bool = False,
     config: bool = False,
     get: bool = False,
     getconfig: bool = False,
     edit: bool = False,
     rpc: bool = False,
     rpcreply: bool = False,
     strict: bool = False,
     trusted: bool = False,
     no_yanglib: bool = False,
     rpc_request: Optional[DNode] = None,
     data_tree: Optional[DNode] = None,
 ) -> DNode:
     if self.cdata is None:
         raise RuntimeError("context already destroyed")
     flags = parser_flags(
         data=data,
         config=config,
         get=get,
         getconfig=getconfig,
         edit=edit,
         rpc=rpc,
         rpcreply=rpcreply,
         strict=strict,
         trusted=trusted,
         no_yanglib=no_yanglib,
     )
     fmt = data_format(fmt)
     args = []
     if rpcreply:
         if rpc_request is None:
             raise ValueError(
                 "rpc_request node is required when rpcreply=True")
         args.append(rpc_request.cdata)
     if rpc or rpcreply:
         if data_tree is not None:
             args.append(data_tree.cdata)
         else:
             args.append(ffi.cast("struct lyd_node *", ffi.NULL))
     dnode = lib.lyd_parse_fd(self.cdata, fileobj.fileno(), fmt, flags,
                              *args)
     if not dnode:
         raise self.error("failed to parse data tree")
     return DNode.new(self, dnode)
예제 #12
0
def _grant_handle(conn: Connection, handle: IO, cpid: int) -> None:
    log = logging.getLogger('_grant_handle')
    flags = get_flags(handle)
    modes = handle.mode
    fd = handle.fileno()
    wrapper = (flags, modes)
    conn.send(wrapper)
    if msvcrt:
        fd = msvcrt.get_osfhandle(fd)
        log.info(
            'Granting Windows-style file handle for {} (fd={}; flags={:02X}; mode={}) to pid {}.'
            .format(handle.name, fd, wrapper[0], handle.mode, cpid))
    else:
        log.info(
            'Granting C-style file descriptor for {} (fd={}; flags={:02X}; mode={}) to pid {}.'
            .format(handle.name, fd, wrapper[0], handle.mode, cpid))
    send_handle(conn, fd, cpid)
예제 #13
0
    def __init__(
        self,
        f: IO,
        encoding: str = "utf-8",
        errors: str = "strict",
        decode: bool = True,
    ) -> None:
        self.encoding = encoding
        self.errors = errors
        self.decoder = codecs.getincrementaldecoder(encoding)(errors=errors)
        self.decode = decode

        self.attributes = {}
        for attribute in ("seekable", "readable"):
            if not hasattr(f, attribute):
                continue
            self.attributes[attribute] = getattr(f, attribute)()
        self.mmap = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
예제 #14
0
 def print_file(
     self,
     fileobj: IO,
     fmt: str,
     with_siblings: bool = False,
     pretty: bool = False,
     include_implicit_defaults: bool = False,
     trim_default_values: bool = False,
     keep_empty_containers: bool = False,
 ) -> None:
     flags = printer_flags(
         with_siblings=with_siblings,
         pretty=pretty,
         include_implicit_defaults=include_implicit_defaults,
         trim_default_values=trim_default_values,
         keep_empty_containers=keep_empty_containers,
     )
     fmt = data_format(fmt)
     ret = lib.lyd_print_fd(fileobj.fileno(), self.cdata, fmt, flags)
     if ret != 0:
         raise self.context.error("cannot print node")
예제 #15
0
def peek_filelike_length(stream: typing.IO) -> int:
    """
    Given a file-like stream object, return its length in number of bytes
    without reading it into memory.
    """
    try:
        # Is it an actual file?
        fd = stream.fileno()
    except OSError:
        # No... Maybe it's something that supports random access, like `io.BytesIO`?
        try:
            # Assuming so, go to end of stream to figure out its length,
            # then put it back in place.
            offset = stream.tell()
            length = stream.seek(0, os.SEEK_END)
            stream.seek(offset)
        except OSError:
            # Not even that? Sorry, we're doomed...
            raise
        else:
            return length
    else:
        # Yup, seems to be an actual file.
        return os.fstat(fd).st_size
예제 #16
0
    def lock(file_: typing.IO, flags: constants.LockFlags):
        if flags & constants.LockFlags.SHARED:
            if sys.version_info.major == 2:
                if flags & constants.LockFlags.NON_BLOCKING:
                    mode = win32con.LOCKFILE_FAIL_IMMEDIATELY
                else:
                    mode = 0

            else:
                if flags & constants.LockFlags.NON_BLOCKING:
                    mode = msvcrt.LK_NBRLCK
                else:
                    mode = msvcrt.LK_RLCK

            # is there any reason not to reuse the following structure?
            hfile = win32file._get_osfhandle(file_.fileno())
            try:
                win32file.LockFileEx(hfile, mode, 0, -0x10000, __overlapped)
            except pywintypes.error as exc_value:
                # error: (33, 'LockFileEx', 'The process cannot access the file
                # because another process has locked a portion of the file.')
                if exc_value.winerror == winerror.ERROR_LOCK_VIOLATION:
                    raise exceptions.LockException(
                        exceptions.LockException.LOCK_FAILED,
                        exc_value.strerror,
                        fh=file_)
                else:
                    # Q:  Are there exceptions/codes we should be dealing with
                    # here?
                    raise
        else:
            if flags & constants.LockFlags.NON_BLOCKING:
                mode = msvcrt.LK_NBLCK
            else:
                mode = msvcrt.LK_LOCK

            # windows locks byte ranges, so make sure to lock from file start
            try:
                savepos = file_.tell()
                if savepos:
                    # [ ] test exclusive lock fails on seek here
                    # [ ] test if shared lock passes this point
                    file_.seek(0)
                    # [x] check if 0 param locks entire file (not documented in
                    #     Python)
                    # [x] fails with "IOError: [Errno 13] Permission denied",
                    #     but -1 seems to do the trick

                try:
                    msvcrt.locking(file_.fileno(), mode, lock_length)
                except IOError as exc_value:
                    # [ ] be more specific here
                    raise exceptions.LockException(
                        exceptions.LockException.LOCK_FAILED,
                        exc_value.strerror,
                        fh=file_)
                finally:
                    if savepos:
                        file_.seek(savepos)
            except IOError as exc_value:
                raise exceptions.LockException(
                    exceptions.LockException.LOCK_FAILED,
                    exc_value.strerror,
                    fh=file_)
예제 #17
0
파일: common.py 프로젝트: bhegerle/pandas
 def __init__(self, f: IO):
     self.mmap = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
예제 #18
0
 def lock(file: IO, flags: int) -> None:
     """Lock a file with flags."""
     hfile = win32file._get_osfhandle(  # pylint: disable=protected-access
         file.fileno())
     win32file.LockFileEx(hfile, flags, 0, 0xFFFF0000, __overlapped)
예제 #19
0
	def large_file(self, file: IO) -> bool:
		file_size = os.fstat(file.fileno()).st_size
		if file_size > self._allowed_size:
			return True
		return False
예제 #20
0
 def file_nsid(file: IO) -> int:
     """Return the namespace identifier (inode number) of the
     namespace specified by a file."""
     return os.stat(file.fileno()).st_ino
예제 #21
0
def _ioctl_uint32(device_file: IO, request: int) -> int:
    buf = b"\0" * 4
    buf = fcntl.ioctl(device_file.fileno(), request, buf)  # type: ignore
    result = struct.unpack("I", buf)[0]
    assert result > 0, (device_file, request, buf)
    return result
예제 #22
0
 def unlock(file: IO) -> None:
     """Unlock a file."""
     hfile = win32file._get_osfhandle(  # pylint: disable=protected-access
         file.fileno())
     win32file.UnlockFileEx(hfile, 0, 0xFFFF0000, __overlapped)
예제 #23
0
 def lock(file: IO, flags: int) -> None:
     """Lock a file with flags."""
     fcntl.flock(file.fileno(), flags)
예제 #24
0
def _maybe_close_fd(fh: IO) -> None:
    try:
        os.close(fh.fileno())
    except (AttributeError, OSError, TypeError):
        # TypeError added for celery#962
        pass
예제 #25
0
 def unlock(file: IO) -> None:
     """Unlock a file."""
     fcntl.flock(file.fileno(), fcntl.LOCK_UN)
예제 #26
0
 def unlock(file_: typing.IO, ):
     fcntl.flock(file_.fileno(), constants.LockFlags.UNBLOCK)