def is_stdin(file: IO) -> bool:
try:
file_no = file.fileno()
except Exception:
return False
else:
return file_no == sys.stdin.fileno()
def upload_file(self, file_handle: IO, file_url: str) -> None:
assert self.project_details
content_type = magic.from_descriptor(file_handle.fileno(), mime=True)
headers = {
"x-amz-acl":
"private" if self.project_details["private"] else "public-read",
"Content-Type": content_type,
}
# This is weird but requests handles an empty file object
# differently than None, with empty file
# it appends Content-Length: 0 header which triggers 501 on S3
if not file_handle.read(1):
data = None
else:
data = file_handle
file_handle.seek(0)
try:
response = requests.put(file_url, data=data, headers=headers)
response.raise_for_status()
except HTTPError as e:
raise FileUploadFailed(
f"There was an error uploading the file. Error code: {response.status_code}"
) from e
file_name = os.path.basename(file_handle.name)
print(f"File {file_name} uploaded to s3")
def __init__(self, f: IO):
self.attributes = {}
for attribute in ("seekable", "readable", "writeable"):
if not hasattr(f, attribute):
continue
self.attributes[attribute] = getattr(f, attribute)()
self.mmap = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
def print_file(
self, fileobj: IO, fmt: str = "tree", path: Optional[str] = None
) -> None:
fmt = schema_out_format(fmt)
ret = lib.lys_print_fd(fileobj.fileno(), self.cdata, fmt, str2c(path), 0, 0)
if ret != 0:
raise self.context.error("cannot print module")
def parse_module_file(self, fileobj: IO, fmt: str = "yang") -> Module:
if self.cdata is None:
raise RuntimeError("context already destroyed")
fmt = schema_in_format(fmt)
mod = lib.lys_parse_fd(self.cdata, fileobj.fileno(), fmt)
if not mod:
raise self.error("cannot parse module")
return Module(self, mod)
def unlock(file_: typing.IO):
try:
savepos = file_.tell()
if savepos:
file_.seek(0)
try:
msvcrt.locking(file_.fileno(), constants.LockFlags.UNBLOCK,
lock_length)
except IOError as exc:
exception = exc
if exc.strerror == 'Permission denied':
hfile = win32file._get_osfhandle(file_.fileno())
try:
win32file.UnlockFileEx(hfile, 0, -0x10000,
__overlapped)
except pywintypes.error as exc:
exception = exc
if exc.winerror == winerror.ERROR_NOT_LOCKED:
# error: (158, 'UnlockFileEx',
# 'The segment is already unlocked.')
# To match the 'posix' implementation, silently
# ignore this error
pass
else:
# Q: Are there exceptions/codes we should be
# dealing with here?
raise
else:
raise exceptions.LockException(
exceptions.LockException.LOCK_FAILED,
exception.strerror,
fh=file_)
finally:
if savepos:
file_.seek(savepos)
except IOError as exc:
raise exceptions.LockException(
exceptions.LockException.LOCK_FAILED, exc.strerror, fh=file_)
def lock(file_: typing.IO, flags: constants.LockFlags):
locking_exceptions = IOError,
try: # pragma: no cover
locking_exceptions += BlockingIOError, # type: ignore
except NameError: # pragma: no cover
pass
try:
fcntl.flock(file_.fileno(), flags)
except locking_exceptions as exc_value:
# The exception code varies on different systems so we'll catch
# every IO error
raise exceptions.LockException(exc_value, fh=file_)
def flush_file_handle(file_handle: IO):
"""
Attempts to flush the handle. First tries to call .flush and then
tries to apply `os.fsync` to the file descriptor to truly flush the
buffer to the file.
:param file_handle: The file handle to attempt to flush
:type file_handle: IO
"""
if hasattr(file_handle, 'flush'):
file_handle.flush()
if hasattr(file_handle, 'fileno'):
os.fsync(file_handle.fileno())
def get_file_name(file: t.IO) -> str:
"""Get the path of `file`."""
handle = msvcrt.get_osfhandle(file.fileno())
buffer = ctypes.create_unicode_buffer(wintypes.MAX_PATH)
ret_val = GetFinalPathNameByHandle(handle, buffer, wintypes.MAX_PATH, 0)
if ret_val > wintypes.MAX_PATH:
buffer = ctypes.create_unicode_buffer(ret_val)
ret_val = GetFinalPathNameByHandle(handle, buffer, ret_val, 0)
if ret_val == 0:
raise ctypes.WinError()
return buffer.value.removeprefix("\\\\?\\")
def silence(fd: IO):
"""Silence any output from fd."""
from os import close, dup, dup2, fdopen, pipe
# Do not silence when debugging.
if not DEBUG:
# Backup the file
old_fd = fd
# Flush the file so it can be silenced properly.
fd.flush()
# Create a duplicate of fd
new_fd = dup(fd.fileno())
# Create a pipe to write to.
read, write = pipe()
# Set the write to the fd filenumber
dup2(write, fd.fileno())
# Close the pipe.
close(write)
close(read)
# Set fd to the new fd
fd = fdopen(new_fd, 'w')
try:
# Run the commands in the 'with' statement.
yield
finally:
if not DEBUG:
# Return the fd back to its original state.
dup2(fd.fileno(), old_fd.fileno())
fd = old_fd
def parse_data_file(
self,
fileobj: IO,
fmt: str,
data: bool = False,
config: bool = False,
get: bool = False,
getconfig: bool = False,
edit: bool = False,
rpc: bool = False,
rpcreply: bool = False,
strict: bool = False,
trusted: bool = False,
no_yanglib: bool = False,
rpc_request: Optional[DNode] = None,
data_tree: Optional[DNode] = None,
) -> DNode:
if self.cdata is None:
raise RuntimeError("context already destroyed")
flags = parser_flags(
data=data,
config=config,
get=get,
getconfig=getconfig,
edit=edit,
rpc=rpc,
rpcreply=rpcreply,
strict=strict,
trusted=trusted,
no_yanglib=no_yanglib,
)
fmt = data_format(fmt)
args = []
if rpcreply:
if rpc_request is None:
raise ValueError(
"rpc_request node is required when rpcreply=True")
args.append(rpc_request.cdata)
if rpc or rpcreply:
if data_tree is not None:
args.append(data_tree.cdata)
else:
args.append(ffi.cast("struct lyd_node *", ffi.NULL))
dnode = lib.lyd_parse_fd(self.cdata, fileobj.fileno(), fmt, flags,
*args)
if not dnode:
raise self.error("failed to parse data tree")
return DNode.new(self, dnode)
def _grant_handle(conn: Connection, handle: IO, cpid: int) -> None:
log = logging.getLogger('_grant_handle')
flags = get_flags(handle)
modes = handle.mode
fd = handle.fileno()
wrapper = (flags, modes)
conn.send(wrapper)
if msvcrt:
fd = msvcrt.get_osfhandle(fd)
log.info(
'Granting Windows-style file handle for {} (fd={}; flags={:02X}; mode={}) to pid {}.'
.format(handle.name, fd, wrapper[0], handle.mode, cpid))
else:
log.info(
'Granting C-style file descriptor for {} (fd={}; flags={:02X}; mode={}) to pid {}.'
.format(handle.name, fd, wrapper[0], handle.mode, cpid))
send_handle(conn, fd, cpid)
def __init__(
self,
f: IO,
encoding: str = "utf-8",
errors: str = "strict",
decode: bool = True,
) -> None:
self.encoding = encoding
self.errors = errors
self.decoder = codecs.getincrementaldecoder(encoding)(errors=errors)
self.decode = decode
self.attributes = {}
for attribute in ("seekable", "readable"):
if not hasattr(f, attribute):
continue
self.attributes[attribute] = getattr(f, attribute)()
self.mmap = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
def print_file(
self,
fileobj: IO,
fmt: str,
with_siblings: bool = False,
pretty: bool = False,
include_implicit_defaults: bool = False,
trim_default_values: bool = False,
keep_empty_containers: bool = False,
) -> None:
flags = printer_flags(
with_siblings=with_siblings,
pretty=pretty,
include_implicit_defaults=include_implicit_defaults,
trim_default_values=trim_default_values,
keep_empty_containers=keep_empty_containers,
)
fmt = data_format(fmt)
ret = lib.lyd_print_fd(fileobj.fileno(), self.cdata, fmt, flags)
if ret != 0:
raise self.context.error("cannot print node")
def peek_filelike_length(stream: typing.IO) -> int:
"""
Given a file-like stream object, return its length in number of bytes
without reading it into memory.
"""
try:
# Is it an actual file?
fd = stream.fileno()
except OSError:
# No... Maybe it's something that supports random access, like `io.BytesIO`?
try:
# Assuming so, go to end of stream to figure out its length,
# then put it back in place.
offset = stream.tell()
length = stream.seek(0, os.SEEK_END)
stream.seek(offset)
except OSError:
# Not even that? Sorry, we're doomed...
raise
else:
return length
else:
# Yup, seems to be an actual file.
return os.fstat(fd).st_size
def lock(file_: typing.IO, flags: constants.LockFlags):
if flags & constants.LockFlags.SHARED:
if sys.version_info.major == 2:
if flags & constants.LockFlags.NON_BLOCKING:
mode = win32con.LOCKFILE_FAIL_IMMEDIATELY
else:
mode = 0
else:
if flags & constants.LockFlags.NON_BLOCKING:
mode = msvcrt.LK_NBRLCK
else:
mode = msvcrt.LK_RLCK
# is there any reason not to reuse the following structure?
hfile = win32file._get_osfhandle(file_.fileno())
try:
win32file.LockFileEx(hfile, mode, 0, -0x10000, __overlapped)
except pywintypes.error as exc_value:
# error: (33, 'LockFileEx', 'The process cannot access the file
# because another process has locked a portion of the file.')
if exc_value.winerror == winerror.ERROR_LOCK_VIOLATION:
raise exceptions.LockException(
exceptions.LockException.LOCK_FAILED,
exc_value.strerror,
fh=file_)
else:
# Q: Are there exceptions/codes we should be dealing with
# here?
raise
else:
if flags & constants.LockFlags.NON_BLOCKING:
mode = msvcrt.LK_NBLCK
else:
mode = msvcrt.LK_LOCK
# windows locks byte ranges, so make sure to lock from file start
try:
savepos = file_.tell()
if savepos:
# [ ] test exclusive lock fails on seek here
# [ ] test if shared lock passes this point
file_.seek(0)
# [x] check if 0 param locks entire file (not documented in
# Python)
# [x] fails with "IOError: [Errno 13] Permission denied",
# but -1 seems to do the trick
try:
msvcrt.locking(file_.fileno(), mode, lock_length)
except IOError as exc_value:
# [ ] be more specific here
raise exceptions.LockException(
exceptions.LockException.LOCK_FAILED,
exc_value.strerror,
fh=file_)
finally:
if savepos:
file_.seek(savepos)
except IOError as exc_value:
raise exceptions.LockException(
exceptions.LockException.LOCK_FAILED,
exc_value.strerror,
fh=file_)
def __init__(self, f: IO):
self.mmap = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
def lock(file: IO, flags: int) -> None:
"""Lock a file with flags."""
hfile = win32file._get_osfhandle( # pylint: disable=protected-access
file.fileno())
win32file.LockFileEx(hfile, flags, 0, 0xFFFF0000, __overlapped)
def large_file(self, file: IO) -> bool: file_size = os.fstat(file.fileno()).st_size if file_size > self._allowed_size: return True return False
def file_nsid(file: IO) -> int:
"""Return the namespace identifier (inode number) of the
namespace specified by a file."""
return os.stat(file.fileno()).st_ino
def _ioctl_uint32(device_file: IO, request: int) -> int:
buf = b"\0" * 4
buf = fcntl.ioctl(device_file.fileno(), request, buf) # type: ignore
result = struct.unpack("I", buf)[0]
assert result > 0, (device_file, request, buf)
return result
def unlock(file: IO) -> None:
"""Unlock a file."""
hfile = win32file._get_osfhandle( # pylint: disable=protected-access
file.fileno())
win32file.UnlockFileEx(hfile, 0, 0xFFFF0000, __overlapped)
def lock(file: IO, flags: int) -> None:
"""Lock a file with flags."""
fcntl.flock(file.fileno(), flags)
def _maybe_close_fd(fh: IO) -> None:
try:
os.close(fh.fileno())
except (AttributeError, OSError, TypeError):
# TypeError added for celery#962
pass
def unlock(file: IO) -> None:
"""Unlock a file."""
fcntl.flock(file.fileno(), fcntl.LOCK_UN)
def unlock(file_: typing.IO, ):
fcntl.flock(file_.fileno(), constants.LockFlags.UNBLOCK)