def ReadFiles(FileSize, location):
if FileSize == '2k':
NumFiles = No2kFiles
ddBlockSize = 512
elif FileSize == '4k':
NumFiles = No4kFiles
ddBlockSize = '1k'
elif FileSize == '8k':
NumFiles = No8kFiles
ddBlockSize = '2k'
elif FileSize == '16k':
NumFiles = No16kFiles
ddBlockSize = '4k'
elif FileSize == '32k':
NumFiles = No32kFiles
ddBlockSize = '8k'
else:
print "Nonsense"
i = 1
while (i <= NumFiles):
#FSTAT
path = '%s/file%s-%s' % (location, FileSize, i)
print path
fd = os.open(path, os.O_RDWR)
print "FD is %s" % (fd)
os.fstatvfs(fd)
os.close(fd)
sleep(1)
#GETATTR, ACCESS, READ
call("dd if=%s/file%s-%s of=/dev/null bs=%s count=4" %
(location, FileSize, i, ddBlockSize),
shell=True)
i = i + 1
def test_fstatvfs():
if not hasattr(os, 'fstatvfs'):
py.test.skip('posix specific function')
try:
os.fstatvfs(0)
except OSError, e:
py.test.skip("the underlying os.fstatvfs() failed: %s" % e)
def fstatvfs(space, fd):
try:
st = os.fstatvfs(fd)
except OSError as e:
raise wrap_oserror(space, e)
else:
return build_statvfs_result(space, st)
def fstatvfs(space, fd):
try:
st = os.fstatvfs(fd)
except OSError as e:
raise wrap_oserror(space, e)
else:
return build_statvfs_result(space, st)
def _init_local(self):
base_tmp_dir = self.shared_tmp_dir
random_id = '%016x' % random.getrandbits(64)
task_name = random_id + self.task_id
# If any parameters are directories, if we don't
# replace the separators on *nix, it'll create a weird nested directory
task_name = task_name.replace("/", "::")
# Max filename length
max_filename_length = os.fstatvfs(0).f_namemax
self.tmp_dir = os.path.join(base_tmp_dir, task_name[:max_filename_length])
LOGGER.info("Tmp dir: %s", self.tmp_dir)
os.makedirs(self.tmp_dir)
# Dump the code to be run into a pickle file
LOGGER.debug("Dumping pickled class")
self._dump(self.tmp_dir)
# Make sure that all the class's dependencies are tarred and available
LOGGER.debug("Tarballing dependencies")
# Grab luigi and the module containing the code to be run
packages = [luigi, __import__(self.__module__, None, None, 'dummy')]
create_packages_archive(packages, os.path.join(self.tmp_dir, "packages.tar"))
# Now, pass onto the class's specified init_local() method.
self.init_local()
def __call__(self, fd, mode, offset, length):
"""
生成大文件
:param fd:
:param mode:
:param offset:
:param length:
:return:
1, fallocate = Fallocate(False)
f = open("/tmp/f_test.data", "wb")
print(fallocate(f, 0, 0, 2 * 1024 * 1024 * 1024))
2, fallocate = Fallocate.instance()
f = open("/tmp/f_test.data", "wb")
print(fallocate(f, 0, 0, 2 * 1024 * 1024 * 1024))
"""
fd = getattr(fd, "fileno", lambda: fd)()
if self.check_free:
st = os.fstatvfs(fd)
free = st.f_frsize * st.f_bavail - length
if float(free) <= 0:
raise OSError(errno.ENOSPC, "fallocate fail free <= 0")
c_length = ctypes.c_uint64(length)
args = {
'fallocate': (fd, mode, offset, c_length),
'posix_fallocate': (fd, offset, c_length)
}
return self.fallocate(*args[self.func_name])
def _get_plasma_size():
fd = os.open(options.worker.plasma_dir, os.O_RDONLY)
stats = os.fstatvfs(fd)
os.close(fd)
size = stats.f_bsize * stats.f_bavail
# keep some safety margin for allocator fragmentation
return 8 * size // 10
def _init_local(self):
# Set up temp folder in shared directory (trim to max filename length)
base_tmp_dir = self.shared_tmp_dir
random_id = '%016x' % random.getrandbits(64)
folder_name = self.task_id + '-' + random_id
self.tmp_dir = os.path.join(base_tmp_dir, folder_name)
max_filename_length = os.fstatvfs(0).f_namemax
self.tmp_dir = self.tmp_dir[:max_filename_length]
logger.info("Tmp dir: %s", self.tmp_dir)
os.makedirs(self.tmp_dir)
# Dump the code to be run into a pickle file
logging.debug("Dumping pickled class")
self._dump(self.tmp_dir)
if not self.no_tarball:
# Make sure that all the class's dependencies are tarred and available
# This is not necessary if luigi is importable from the cluster node
logging.debug("Tarballing dependencies")
# Grab luigi and the module containing the code to be run
packages = [luigi
] + [__import__(self.__module__, None, None, 'dummy')]
luigi.hadoop.create_packages_archive(
packages, os.path.join(self.tmp_dir, "packages.tar"))
def calc_disk(tag_info):
'''
Calculate disk related statistics
:param tag_info: The dictionary used to hold the tag metadata and
values.
:type tag_info: dict(str, dict)
'''
if salt.utils.platform.is_windows():
root_disk_path = 'C:\\'
else:
root_disk_path = '/'
if six.PY3:
disk_usage = shutil.disk_usage(root_disk_path) # pylint: disable=no-member
disk_total = float(disk_usage.total)
disk_used = float(disk_usage.used)
disk_free = float(disk_usage.free)
else:
fd_ = os.open(root_disk_path, os.O_RDONLY)
try:
disk_info = os.fstatvfs(fd_) # pylint: disable=no-member
disk_total = float(disk_info.f_frsize) * float(disk_info.f_blocks)
disk_free = float(disk_info.f_frsize) * float(disk_info.f_bfree)
disk_used = disk_total - disk_free
finally:
os.close(fd_)
tag_info['disk_free']['value'] = disk_free
tag_info['disk_used']['value'] = disk_used
tag_info['disk_total']['value'] = disk_total
tag_info['disk_perc']['value'] = (disk_used / disk_total) * 100
def user_freespace_gb():
"""
Returns the amount of free space user space in GB
"""
fd = os.open('.', os.O_RDONLY | os.O_DIRECTORY)
info = os.fstatvfs(fd)
os.close(fd)
return (info.f_bsize * info.f_bavail) / (1024 * 1024 * 1024)
def statfs(path):
fd = os.open(path, os.O_RDONLY)
stv = os.fstatvfs(fd)
os.close(fd)
return dict(
(key, getattr(stv, key))
for key in ('f_bavail', 'f_bfree', 'f_blocks', 'f_bsize', 'f_favail',
'f_ffree', 'f_files', 'f_flag', 'f_frsize', 'f_namemax'))
def __init__(self, name):
self.name = name
fd = os.open(self.name, os.O_RDONLY)
disk = os.fstatvfs(fd)
self.totalBytes = float(disk.f_bsize * disk.f_blocks)
self.totalUsed = float(disk.f_bsize * (disk.f_blocks - disk.f_bfree))
self.totalAvail = float(disk.f_bsize * disk.f_bfree)
os.close(fd)
def upload(self, batch_id, file_path, filename=None, file_index=0):
"""Upload a file through an Automation batch
Uses poster.httpstreaming to stream the upload
and not load the whole file in memory.
"""
# Request URL
url = self.automation_url.encode('ascii') + self.batch_upload_url
# HTTP headers
if filename is None:
filename = os.path.basename(file_path)
file_size = os.path.getsize(file_path)
ctype, _ = mimetypes.guess_type(filename)
if ctype:
mime_type = ctype
else:
mime_type = "application/octet-stream"
# Quote UTF-8 filenames even though JAX-RS does not seem to be able
# to retrieve them as per: https://tools.ietf.org/html/rfc5987
filename = safe_filename(filename)
quoted_filename = urllib2.quote(filename.encode('utf-8'))
headers = {
"X-Batch-Id": batch_id,
"X-File-Idx": file_index,
"X-File-Name": quoted_filename,
"X-File-Size": file_size,
"X-File-Type": mime_type,
"Content-Type": "application/octet-stream",
"Content-Length": file_size,
}
headers.update(self._get_common_headers())
# Request data
input_file = open(file_path, 'rb')
# Use file system block size if available for streaming buffer
if sys.platform != 'win32':
fs_block_size = os.fstatvfs(input_file.fileno()).f_bsize
else:
fs_block_size = DEFAULT_STREAMING_BUFFER_SIZE
log.trace("Using file system block size"
" for the streaming upload buffer: %u bytes", fs_block_size)
data = self._read_data(input_file, fs_block_size)
# Execute request
cookies = self._get_cookies()
log.trace("Calling %s with headers %r and cookies %r for file %s",
url, headers, cookies, file_path)
req = urllib2.Request(url, data, headers)
try:
resp = self.streaming_opener.open(req, timeout=self.blob_timeout)
except Exception as e:
self._log_details(e)
raise
finally:
input_file.close()
return self._read_response(resp, url)
def start_objstore(node_ip_address, redis_address, object_manager_port=None,
cleanup=True, redirect_output=False, objstore_memory=None):
"""This method starts an object store process.
Args:
node_ip_address (str): The IP address of the node running the object store.
redis_address (str): The address of the Redis instance to connect to.
object_manager_port (int): The port to use for the object manager. If this
is not provided, one will be generated randomly.
cleanup (bool): True if using Ray in local mode. If cleanup is true, then
this process will be killed by serices.cleanup() when the Python process
that imported services exits.
redirect_output (bool): True if stdout and stderr should be redirected to
/dev/null.
Return:
A tuple of the Plasma store socket name, the Plasma manager socket name, and
the plasma manager port.
"""
if objstore_memory is None:
# Compute a fraction of the system memory for the Plasma store to use.
system_memory = psutil.virtual_memory().total
if sys.platform == "linux" or sys.platform == "linux2":
# On linux we use /dev/shm, its size is half the size of the physical
# memory. To not overflow it, we set the plasma memory limit to 0.4 times
# the size of the physical memory.
objstore_memory = int(system_memory * 0.4)
# Compare the requested memory size to the memory available in /dev/shm.
shm_fd = os.open("/dev/shm", os.O_RDONLY)
try:
shm_fs_stats = os.fstatvfs(shm_fd)
# The value shm_fs_stats.f_bsize is the block size and the value
# shm_fs_stats.f_bavail is the number of available blocks.
shm_avail = shm_fs_stats.f_bsize * shm_fs_stats.f_bavail
if objstore_memory > shm_avail:
print("Warning: Reducing object store memory because /dev/shm has only {} bytes available. You may be able to free up space by deleting files in /dev/shm. If you are inside a Docker container, you may need to pass an argument with the flag '--shm-size' to 'docker run'.".format(shm_avail))
objstore_memory = int(shm_avail * 0.8)
finally:
os.close(shm_fd)
else:
objstore_memory = int(system_memory * 0.8)
# Start the Plasma store.
plasma_store_name, p1 = plasma.start_plasma_store(plasma_store_memory=objstore_memory, use_profiler=RUN_PLASMA_STORE_PROFILER, redirect_output=redirect_output)
# Start the plasma manager.
if object_manager_port is not None:
plasma_manager_name, p2, plasma_manager_port = plasma.start_plasma_manager(plasma_store_name, redis_address, plasma_manager_port=object_manager_port, node_ip_address=node_ip_address, num_retries=1, run_profiler=RUN_PLASMA_MANAGER_PROFILER, redirect_output=redirect_output)
assert plasma_manager_port == object_manager_port
else:
plasma_manager_name, p2, plasma_manager_port = plasma.start_plasma_manager(plasma_store_name, redis_address, node_ip_address=node_ip_address, run_profiler=RUN_PLASMA_MANAGER_PROFILER, redirect_output=redirect_output)
if cleanup:
all_processes[PROCESS_TYPE_PLASMA_STORE].append(p1)
all_processes[PROCESS_TYPE_PLASMA_MANAGER].append(p2)
return ObjectStoreAddress(plasma_store_name, plasma_manager_name,
plasma_manager_port)
def _init_local(self):
# Set up temp folder in shared directory (trim to max filename length)
base_tmp_dir = self.shared_tmp_dir
random_id = '%016x' % random.getrandbits(64)
folder_name = self.task_id + '-' + random_id
self.tmp_dir = os.path.join(base_tmp_dir, folder_name)
max_filename_length = os.fstatvfs(0).f_namemax
self.tmp_dir = self.tmp_dir[:max_filename_length]
logger.info("Tmp dir: %s", self.tmp_dir)
os.makedirs(self.tmp_dir)
# Dump the code to be run into a pickle file
self._dump(self.tmp_dir)
def _get_plasma_limit(self):
if sys.platform == 'win32': # pragma: no cover
return
elif sys.platform == 'darwin':
default_plasma_dir = '/tmp'
else:
default_plasma_dir = '/dev/shm'
fd = os.open(self._plasma_dir or default_plasma_dir, os.O_RDONLY)
stats = os.fstatvfs(fd)
os.close(fd)
size = stats.f_bsize * stats.f_bavail
# keep some safety margin for allocator fragmentation
return 8 * size // 10
def _get_io_blocksize_MB_(pathname):
"""
Return the I/O blocksize for a given path (used to estimate IOPS)
"""
if isfile(pathname):
fname = pathname
fflag = O_RDONLY
elif isdir(pathname):
fname = join(pathname, '.bsize_test_file')
fflag = O_CREAT
else:
return None
fd = fdopen(fname, fflag)
bsize = fstatvfs(fd).f_bsize
fdclose(fd)
return bsize
def _get_io_blocksize_MB_(pathname):
"""
Return the I/O blocksize for a given path (used to estimate IOPS)
"""
if isfile(pathname):
fname = pathname
fflag = O_RDONLY
elif isdir(pathname):
fname = join(pathname, '.bsize_test_file')
fflag = O_CREAT
else:
return None
fd = fdopen(fname, fflag)
bsize = fstatvfs(fd).f_bsize
fdclose(fd)
return bsize
def func_diskspace(self, *args):
"""
diskspace("/var") -> 125 (12.5% used)
"""
if len(args) != 1:
log.warning("Warning: function diskspace takes exactly 1 argument.")
return
mountpoint = args[0]
fd = os.open(mountpoint, os.O_RDONLY)
stats = os.fstatvfs(fd)
os.close(fd)
size = stats[1] * stats[2]
free = stats[1] * stats[4]
diskspace = 1000 * (size - free) / size
return diskspace
def check_free(self):
if self.check_free_blocks == -1:
return False
fst = os.fstatvfs(self.out_fd.fileno())
if not self.check_free_blocks and config.has_section("free"):
best = None
for ent in config.options("free"):
if self.output_real[0:len(ent)] != ent:
continue
if best and len(ent) > len(best):
best = ent
else:
best = ent
# no match at all ?
if not best:
return True
# calculate the free block counter
should = config.get("free", best)
# percentage
if should in ('off', 'disabled', 'disable', 'none'):
self.check_free_blocks = -1
return False
elif should[-1] == "%":
self.check_free_blocks = (fst.f_blocks/100)*int(should[:-1] or 10)
else:
factors = {
"k": 1000,
"m": 1000 ** 2,
"g": 1000 ** 3,
"t": 1000 ** 4,
"p": 1000 ** 5 }
if factors.has_key(should[-1].lower()):
should = int(should[:-1]) * factors[should[-1].lower()]
else:
# default to megabyte
should = int(should) * factors["m"]
self.check_free_blocks = should / fst.f_frsize # should be multiples of f_frsize, i guess
# actual check
if self.check_free_blocks is not None:
if fst.f_bfree < self.check_free_blocks:
return False
return True
def fibmap2(fd, start=0, end=None, flags=0):
xstat = os.fstat(fd)
if not stat.S_ISREG(xstat.st_mode):
return
if flags & FIEMAP_FLAG_XATTR:
return
if end is None:
end = xstat.st_size
statvfs = os.fstatvfs(fd)
block_size = statvfs.f_bsize
start_block = start // block_size
end_block = (end + block_size - 1) // block_size
block = start_block
fe_pblk = None
fe_lblk = None
fe_len = None
max_extent = MAX_EXTENT_LENGTH / block_size
while block <= end_block:
indata = struct.pack('i', block)
res = fcntl.ioctl(fd, _FIBMAP, indata)
pblock = struct.unpack('i', res)[0]
if fe_pblk is not None:
if pblock > 0 and pblock == fe_pblk + fe_len and \
fe_len <= max_extent:
fe_len += 1
else:
yield fiemap_rec(fe_lblk * block_size, \
fe_pblk * block_size, \
fe_len * block_size, 0, \
FIEMAP_EXTENT_MERGED)
fe_pblk = fe_lblk = fe_len = None
else:
if pblock > 0:
fe_pblk = pblock
fe_lblk = block
fe_len = 1
block += 1
if fe_pblk is not None:
yield fiemap_rec(fe_lblk * block_size, \
fe_pblk * block_size, fe_len * block_size, 0, \
FIEMAP_EXTENT_MERGED)
def get_free_disk_space():
''' Returns disk space currently free in MB'''
docker_path = '/var/lib/docker'
try:
fd = os.open(docker_path, os.O_RDONLY)
stats = os.fstatvfs(fd)
bytes_free = stats.f_frsize * stats.f_bavail # block size * number of free blocks
mb_free = bytes_free * 1e-6
return mb_free
except Exception as exc:
utils_logger.debug('Free disk space could not be determined.')
finally:
try:
os.close(fd)
except (NameError, OSError):
# file has not been opened
pass
return
def __call__(
self,
fd,
mode,
offset,
length,
):
""" The length parameter must be a ctypes.c_uint64 """
if FALLOCATE_RESERVE > 0:
st = os.fstatvfs(fd)
free = st.f_frsize * st.f_bavail - length.value
if free <= FALLOCATE_RESERVE:
raise OSError('FALLOCATE_RESERVE fail %s <= %s'
% (free, FALLOCATE_RESERVE))
args = {'fallocate': (fd, mode, offset, length),
'posix_fallocate': (fd, offset, length)}
return self.fallocate(*args[self.func_name])
def check_free(self):
if self.check_free_blocks == -1:
return False
fst = os.fstatvfs(self.out_fd.fileno())
if not self.check_free_blocks and config.has_section("free"):
best = None
for ent in config.options("free"):
if self.output_real[0 : len(ent)] != ent:
continue
if best and len(ent) > len(best):
best = ent
else:
best = ent
# no match at all ?
if not best:
return True
# calculate the free block counter
should = config.get("free", best)
# percentage
if should in ("off", "disabled", "disable", "none"):
self.check_free_blocks = -1
return False
elif should[-1] == "%":
self.check_free_blocks = (fst.f_blocks / 100) * int(should[:-1] or 10)
else:
factors = {"k": 1000, "m": 1000 ** 2, "g": 1000 ** 3, "t": 1000 ** 4, "p": 1000 ** 5}
if factors.has_key(should[-1].lower()):
should = int(should[:-1]) * factors[should[-1].lower()]
else:
# default to megabyte
should = int(should) * factors["m"]
self.check_free_blocks = should / fst.f_frsize # should be multiples of f_frsize, i guess
# actual check
if self.check_free_blocks is not None:
if fst.f_bfree < self.check_free_blocks:
return False
return True
def get_shared_memory_bytes():
"""Get the size of the shared memory file system.
Returns:
The size of the shared memory file system in bytes.
"""
# Make sure this is only called on Linux.
assert sys.platform == "linux" or sys.platform == "linux2"
shm_fd = os.open("/dev/shm", os.O_RDONLY)
try:
shm_fs_stats = os.fstatvfs(shm_fd)
# The value shm_fs_stats.f_bsize is the block size and the
# value shm_fs_stats.f_bavail is the number of available
# blocks.
shm_avail = shm_fs_stats.f_bsize * shm_fs_stats.f_bavail
finally:
os.close(shm_fd)
return shm_avail
def get_shared_memory_bytes():
"""Get the size of the shared memory file system.
Returns:
The size of the shared memory file system in bytes.
"""
# Make sure this is only called on Linux.
assert sys.platform == "linux" or sys.platform == "linux2"
shm_fd = os.open("/dev/shm", os.O_RDONLY)
try:
shm_fs_stats = os.fstatvfs(shm_fd)
# The value shm_fs_stats.f_bsize is the block size and the
# value shm_fs_stats.f_bavail is the number of available
# blocks.
shm_avail = shm_fs_stats.f_bsize * shm_fs_stats.f_bavail
finally:
os.close(shm_fd)
return shm_avail
def __call__(
self,
fd,
mode,
offset,
length,
):
""" The length parameter must be a ctypes.c_uint64 """
if FALLOCATE_RESERVE > 0:
st = os.fstatvfs(fd)
free = st.f_frsize * st.f_bavail - length.value
if free <= FALLOCATE_RESERVE:
raise OSError('FALLOCATE_RESERVE fail %s <= %s' %
(free, FALLOCATE_RESERVE))
args = {
'fallocate': (fd, mode, offset, length),
'posix_fallocate': (fd, offset, length)
}
return self.fallocate(*args[self.func_name])
def _init_local(self):
base_tmp_dir = self.shared_tmp_dir
random_id = "%016x" % random.getrandbits(64)
task_name = random_id + self.task_id
# If any parameters are directories, if we don't
# replace the separators on *nix, it'll create a weird nested directory
task_name = task_name.replace("/", "::")
# Max filename length
max_filename_length = os.fstatvfs(0).f_namemax
self.tmp_dir = os.path.join(base_tmp_dir,
task_name[:max_filename_length])
LOGGER.info("Tmp dir: %s", self.tmp_dir)
os.makedirs(self.tmp_dir)
# Now, pass onto the class's specified init_local() method.
self.init_local()
def _init_local(self):
# Set up temp folder in shared directory (trim to max filename length)
base_tmp_dir = self.shared_tmp_dir
random_id = '%016x' % random.getrandbits(64)
folder_name = _clean_task_id(self.task_id) + '-' + random_id
self.tmp_dir = os.path.join(base_tmp_dir, folder_name)
max_filename_length = os.fstatvfs(0).f_namemax
self.tmp_dir = self.tmp_dir[:max_filename_length]
logger.info("Tmp dir: %s", self.tmp_dir)
os.makedirs(self.tmp_dir)
# Dump the code to be run into a pickle file
logging.debug("Dumping pickled class")
self._dump(self.tmp_dir)
# Make sure that all the class's dependencies are tarred and available
logging.debug("Tarballing dependencies")
# Grab luigi and the module containing the code to be run
packages = [luigi] + [__import__(self.__module__, None, None, 'dummy')]
luigi.hadoop.create_packages_archive(packages, os.path.join(self.tmp_dir, "packages.tar"))
USER=os.getlogin()
print "User="******"GID:UID", GID, ":", UID
print "Creating a temp file and reading/writing it"
tfile=os.tmpfile()
tfile.write("Hello 1\n")
tfile.write("Hello 2\n")
tfile.seek(0)
for x in tfile:
x=x.rstrip()
print x
tfd=tfile.fileno()
tstat=os.fstat(tfd)
print "Stat info on temp file=", tstat
vfsinfo=os.fstatvfs(tfd)
print "VFS info on temp file=", vfsinfo
print "truncating file, then writing new data"
os.ftruncate(tfd,0)
tfile.write("Hello a\n")
tfile.write("Hello b\n")
tfile.seek(0)
for x in tfile:
x=x.rstrip()
print x
tfile.close();
print "Getting list of files in this dir"
# note, list is not sorted so we sort it here
#open a file1
fd = os.open("foo.txt", os.O_RDWR | os.O_CREAT)
#Now get the touple
info = os.fstat(fd)
print "File Info : ", info
print "File UUID : ", info.st_uid
print "File GIS : ", info.st_gid
print "File ctime : ", info.st_ctime
print "File Block Size :", info.st_blksize
print "File protection :", info.st_mode
sys_info = os.fstatvfs(fd)
print "File system info : ", sys_info
print "Maximum filename length : ", sys_info.f_namemax
print "create hard link to file foo.txt"
try:
os.link("foo.txt", "foo1.txt")
except Exception as ex:
print "Link already exist"
print "List current directory's contents : \n",
for file in os.listdir(newdir):
print file
ls_info = os.lstat("foo1.txt")
def get_file(self, user, suite, path, path_in_tar=None, mode=None):
"""Returns file information / content or a cherrypy response."""
f_name = self._get_user_suite_dir(user, suite, path)
self._check_file_path(path)
view_size_max = self.VIEW_SIZE_MAX
if path_in_tar:
tar_f = tarfile.open(f_name, "r:gz")
try:
tar_info = tar_f.getmember(path_in_tar)
except KeyError:
raise cherrypy.HTTPError(404)
f_size = tar_info.size
handle = tar_f.extractfile(path_in_tar)
if handle.read(2) == "#!":
mime = self.MIME_TEXT_PLAIN
else:
mime = mimetypes.guess_type(
urllib.pathname2url(path_in_tar))[0]
handle.seek(0)
if (mode == "download" or f_size > view_size_max or mime and
(not mime.startswith("text/") or mime.endswith("html"))):
temp_f = NamedTemporaryFile()
f_bsize = os.fstatvfs(temp_f.fileno()).f_bsize
while True:
bytes_ = handle.read(f_bsize)
if not bytes_:
break
temp_f.write(bytes_)
cherrypy.response.headers["Content-Type"] = mime
try:
return cherrypy.lib.static.serve_file(temp_f.name, mime)
finally:
temp_f.close()
text = handle.read()
else:
f_size = os.stat(f_name).st_size
if open(f_name).read(2) == "#!":
mime = self.MIME_TEXT_PLAIN
else:
mime = mimetypes.guess_type(urllib.pathname2url(f_name))[0]
if not mime:
mime = self.MIME_TEXT_PLAIN
if (mode == "download" or f_size > view_size_max or mime and
(not mime.startswith("text/") or mime.endswith("html"))):
cherrypy.response.headers["Content-Type"] = mime
return cherrypy.lib.static.serve_file(f_name, mime)
text = open(f_name).read()
try:
if mode in [None, "text"]:
text = jinja2.escape(text)
lines = [unicode(line) for line in text.splitlines()]
except UnicodeDecodeError:
if path_in_tar:
handle.seek(0)
# file closed by cherrypy
return cherrypy.lib.static.serve_fileobj(
handle, self.MIME_TEXT_PLAIN)
else:
return cherrypy.lib.static.serve_file(f_name,
self.MIME_TEXT_PLAIN)
else:
if path_in_tar:
handle.close()
name = path
if path_in_tar:
name = "log/" + path_in_tar
job_entry = None
if name.startswith("log/job"):
names = name.replace("log/job/", "").split("/", 3)
if len(names) == 4:
cycle, task, submit_num, _ = names
entries = self.suite_dao.get_suite_job_entries(
user, suite, [cycle], [task], None, None, None, None,
None)[0]
for entry in entries:
if entry["submit_num"] == int(submit_num):
job_entry = entry
break
if fnmatch(os.path.basename(path), "suite*.rc*"):
file_content = "cylc-suite-rc"
elif fnmatch(os.path.basename(path), "rose*.conf"):
file_content = "rose-conf"
else:
file_content = None
return lines, job_entry, file_content, f_name
def view(self, user, suite, path, path_in_tar=None, mode=None):
"""View a text log file."""
f_name = self._get_user_suite_dir(user, suite, path)
if not os.access(f_name, os.F_OK | os.R_OK):
raise cherrypy.HTTPError(404)
if path_in_tar:
tar_f = tarfile.open(f_name, 'r:gz')
try:
tar_info = tar_f.getmember(path_in_tar)
except KeyError:
raise cherrypy.HTTPError(404)
f_size = tar_info.size
f = tar_f.extractfile(path_in_tar)
if f.read(2) == "#!":
mime = MIME_TEXT_PLAIN
else:
mime = mimetypes.guess_type(
urllib.pathname2url(path_in_tar))[0]
f.seek(0)
if (mode == "download" or
f_size > self.VIEW_SIZE_MAX or
mime and (not mime.startswith("text/") or
mime.endswith("html"))):
t = NamedTemporaryFile()
f_bsize = os.fstatvfs(t.fileno()).f_bsize
while True:
bytes = f.read(f_bsize)
if not bytes:
break
t.write(bytes)
cherrypy.response.headers["Content-Type"] = mime
try:
return cherrypy.lib.static.serve_file(t.name, mime)
finally:
t.close()
s = f.read()
f.close()
else:
f_size = os.stat(f_name).st_size
if open(f_name).read(2) == "#!":
mime = MIME_TEXT_PLAIN
else:
mime = mimetypes.guess_type(urllib.pathname2url(f_name))[0]
if (mode == "download" or
f_size > self.VIEW_SIZE_MAX or
mime and (not mime.startswith("text/") or
mime.endswith("html"))):
cherrypy.response.headers["Content-Type"] = mime
return cherrypy.lib.static.serve_file(f_name, mime)
s = open(f_name).read()
if mode == "text":
s = jinja2.escape(s)
try:
lines = [unicode(line) for line in s.splitlines()]
except UnicodeDecodeError:
return cherrypy.lib.static.serve_file(f_name, MIME_TEXT_PLAIN)
name = path
if path_in_tar:
name = path_in_tar
if fnmatch(os.path.basename(path), "rose*.conf"):
file_content = "rose-conf"
else:
file_content = self.suite_engine_proc.is_conf(path)
template = self.template_env.get_template("view.html")
return template.render(
rose_version=self.rose_version,
script=cherrypy.request.script_name,
time=strftime("%Y-%m-%dT%H:%M:%S+0000", gmtime()),
host=self.host_name,
user=user,
suite=suite,
path=path,
path_in_tar=path_in_tar,
f_name=f_name,
mode=mode,
file_content=file_content,
lines=lines)
def test_fstatvfs(self):
try:
os.fstatvfs(0)
except OSError as e:
py.test.skip("the underlying os.fstatvfs() failed: %s" % e)
rposix_stat.fstatvfs(0)
def get_file(self, user, suite, path, path_in_tar=None, mode=None):
"""Returns file information / content or a cherrypy response."""
f_name = self._get_user_suite_dir(user, suite, path)
conf = ResourceLocator.default().get_conf()
view_size_max = int(conf.get_value(
["rose-bush", "view-size-max"], self.VIEW_SIZE_MAX))
if path_in_tar:
tar_f = tarfile.open(f_name, "r:gz")
try:
tar_info = tar_f.getmember(path_in_tar)
except KeyError:
raise cherrypy.HTTPError(404)
f_size = tar_info.size
handle = tar_f.extractfile(path_in_tar)
if handle.read(2) == "#!":
mime = self.MIME_TEXT_PLAIN
else:
mime = mimetypes.guess_type(
urllib.pathname2url(path_in_tar))[0]
handle.seek(0)
if (mode == "download" or
f_size > view_size_max or
mime and
(not mime.startswith("text/") or mime.endswith("html"))):
temp_f = NamedTemporaryFile()
f_bsize = os.fstatvfs(temp_f.fileno()).f_bsize
while True:
bytes_ = handle.read(f_bsize)
if not bytes_:
break
temp_f.write(bytes_)
cherrypy.response.headers["Content-Type"] = mime
try:
return cherrypy.lib.static.serve_file(temp_f.name, mime)
finally:
temp_f.close()
text = handle.read()
else:
f_size = os.stat(f_name).st_size
if open(f_name).read(2) == "#!":
mime = self.MIME_TEXT_PLAIN
else:
mime = mimetypes.guess_type(urllib.pathname2url(f_name))[0]
if not mime:
mime = self.MIME_TEXT_PLAIN
if (mode == "download" or
f_size > view_size_max or
mime and
(not mime.startswith("text/") or mime.endswith("html"))):
cherrypy.response.headers["Content-Type"] = mime
return cherrypy.lib.static.serve_file(f_name, mime)
text = open(f_name).read()
try:
if mode in [None, "text"]:
text = jinja2.escape(text)
lines = [unicode(line) for line in text.splitlines()]
except UnicodeDecodeError:
if path_in_tar:
handle.seek(0)
# file closed by cherrypy
return cherrypy.lib.static.serve_fileobj(
handle, self.MIME_TEXT_PLAIN)
else:
return cherrypy.lib.static.serve_file(
f_name, self.MIME_TEXT_PLAIN)
else:
if path_in_tar:
handle.close()
name = path
if path_in_tar:
name = "log/" + path_in_tar
job_entry = None
if name.startswith("log/job"):
names = self.bush_dao.parse_job_log_rel_path(name)
if len(names) == 4:
cycle, task, submit_num, _ = names
entries = self.bush_dao.get_suite_job_entries(
user, suite, [cycle], [task],
None, None, None, None, None)[0]
for entry in entries:
if entry["submit_num"] == int(submit_num):
job_entry = entry
break
if fnmatch(os.path.basename(path), "rose*.conf"):
file_content = "rose-conf"
else:
file_content = self.bush_dao.is_conf(path)
return lines, job_entry, file_content, f_name
os.fchown(fd,uid,gid) # 修改一个文件的所有权，这个函数修改一个文件的用户ID和用户组ID，该文件由文件描述符fd指定 os.fdatasync(fd) # 强制将文件谢谢如磁盘，该文件由文件描述符fd指定，但是不强制更新文件的状态信息 os.fdopen(fd[,mode[,bufsize]]) # 通过文件描述符fd创建一个文件独享，并返回这个文件对象 os.fpathconf(fd,name) # 返回一个打开的文件的系统配置信息。name为检索的系统配置的值，它也许是一个定义系统值的字符串，这些名字在很多标准中指定（POSIX.1, Unix 95, Unix 98, 和其它）。 os.fstat(fd) # 返回文件描述符fd的状态，像stat() os.fstatvfs(fd) # 返回包含文件描述符fd的文件的文件系统的信息，想stavfs() os.fsync(fd) # 强制将文件描述符为fd的文件写入硬盘 os.ftruncate(fd,length) # 裁剪文件描述符fd对应的文件，所以它最大不能超过文件大小 os.isatty(fd) # 如果文件描述符fd是打开的，同时与tty(-like)设备相连，则返回True,否则False os.tcgetpgrp(fd) # 返回与终端fd（一个由os.open()返回的打开的文件描述符）关联的进程组 os.tcsetpgrp(fd, pg)
def fstatvfs(self, handle):
fo = self._files.get(handle, None)
if fo:
return os.fstatvfs(fo.fileno())
def _swift_upload(self, swift, conn, infile):
"""Puts the contents of infile into a container on swift. If the file
is larger than 100M, it will be divided into segments using
the "dynamic large objects" convention.
infile is expected to be a pipe, so each segment is stored in
a temporary file before uploading. This allows the
python-swiftclient library to retry failed uploads.
"""
SEGMENT_SIZE = getattr(settings, "SWIFT_BACKEND_SEGMENT_SIZE",
100 * 1024 * 1024)
segment_buf = tempfile.NamedTemporaryFile()
st = os.fstatvfs(segment_buf.fileno())
free_space = st.f_bavail * st.f_bsize
if free_space < SEGMENT_SIZE:
logger.error("There is only %dM space available for %s and %dM is required." % (free_space / 1024 / 1024, segment_buf.name, SEGMENT_SIZE / 1024 / 1024))
def load_segment():
"""Copies enough data for a segment from the fifo into a temporary
file. Returns how much data was actually written. """
written = 0
while written < SEGMENT_SIZE:
bufsize = min(64 * 1024, SEGMENT_SIZE - written)
buf = infile.read(bufsize)
if not buf:
break
segment_buf.write(buf)
written += len(buf)
segment_buf.seek(0)
return written
def upload_segment(bucket, prefix, length):
try:
conn.put_object(bucket, prefix, segment_buf,
content_length=length,
chunk_size=self.CHUNKSIZE)
except swiftclient.exceptions.ClientException:
logger.exception("Error when uploading segment")
raise
except IOError:
logger.exception("Error when uploading segment")
raise
def upload_manifest(bucket, prefix):
headers = {"X-Object-Manifest": "%s_segments/%s/" % (bucket, prefix)}
conn.put_object(bucket, prefix, "", content_length=0,
headers=headers)
seg_num = 0
last_seg = False
while not last_seg:
seg_size = load_segment()
last_seg = seg_size < SEGMENT_SIZE
if seg_num == 0 and last_seg:
bucket = swift.bucket
prefix = swift.prefix
else:
bucket = swift.bucket + "_segments"
prefix = "%s/%s/%08d" % (swift.prefix, SEGMENT_SIZE, seg_num)
if seg_num == 0:
logger.debug("%s %s will be uploaded in %s byte segments." % (swift.bucket, swift.prefix, SEGMENT_SIZE))
# ensure that a segments container exists
conn.put_container(bucket)
upload_segment(bucket, prefix, seg_size)
segment_buf.seek(0)
segment_buf.truncate(0)
seg_num += 1
if seg_num > 1:
upload_manifest(swift.bucket, swift.prefix)
return True
def ino_check(dev, min_ino=MIN_INODES):
return os.fstatvfs(dev).f_ffree >= int(min_ino)
def get_upload_buffer(self, input_file):
if sys.platform != 'win32':
return os.fstatvfs(input_file.fileno()).f_bsize
else:
return FILE_BUFFER_SIZE
def space_check(dev, size):
stdev = os.fstatvfs(dev)
return size < stdev.f_bfree * stdev.f_bsize
def get_upload_buffer(self, input_file):
if sys.platform != 'win32':
return os.fstatvfs(input_file.fileno()).f_bsize
else:
return FILE_BUFFER_SIZE
def initialize_ray(
override_is_cluster=False,
override_redis_address: str = None,
override_redis_password: str = None,
):
"""
Initialize Ray based on parameters, ``modin.config`` variables and internal defaults.
Parameters
----------
override_is_cluster : bool, default: False
Whether to override the detection of Modin being run in a cluster
and always assume this runs on cluster head node.
This also overrides Ray worker detection and always runs the initialization
function (runs from main thread only by default).
If not specified, ``modin.config.IsRayCluster`` variable is used.
override_redis_address : str, optional
What Redis address to connect to when running in Ray cluster.
If not specified, ``modin.config.RayRedisAddress`` is used.
override_redis_password : str, optional
What password to use when connecting to Redis.
If not specified, ``modin.config.RayRedisPassword`` is used.
"""
import ray
if not ray.is_initialized() or override_is_cluster:
cluster = override_is_cluster or IsRayCluster.get()
redis_address = override_redis_address or RayRedisAddress.get()
redis_password = override_redis_password or RayRedisPassword.get()
if cluster:
# We only start ray in a cluster setting for the head node.
ray.init(
address=redis_address or "auto",
include_dashboard=False,
ignore_reinit_error=True,
_redis_password=redis_password,
)
else:
from modin.error_message import ErrorMessage
# This string is intentionally formatted this way. We want it indented in
# the warning message.
ErrorMessage.not_initialized(
"Ray",
"""
import ray
ray.init()
""",
)
object_store_memory = Memory.get()
# In case anything failed above, we can still improve the memory for Modin.
if object_store_memory is None:
virtual_memory = psutil.virtual_memory().total
if sys.platform.startswith("linux"):
shm_fd = os.open("/dev/shm", os.O_RDONLY)
try:
shm_stats = os.fstatvfs(shm_fd)
system_memory = shm_stats.f_bsize * shm_stats.f_bavail
if system_memory / (virtual_memory / 2) < 0.99:
warnings.warn(
f"The size of /dev/shm is too small ({system_memory} bytes). The required size "
f"at least half of RAM ({virtual_memory // 2} bytes). Please, delete files in /dev/shm or "
"increase size of /dev/shm with --shm-size in Docker. Also, you can set "
"the required memory size for each Ray worker in bytes to MODIN_MEMORY environment variable."
)
finally:
os.close(shm_fd)
else:
system_memory = virtual_memory
object_store_memory = int(0.6 * system_memory // 1e9 * 1e9)
# If the memory pool is smaller than 2GB, just use the default in ray.
if object_store_memory == 0:
object_store_memory = None
else:
object_store_memory = int(object_store_memory)
ray_init_kwargs = {
"num_cpus": CpuCount.get(),
"num_gpus": GpuCount.get(),
"include_dashboard": False,
"ignore_reinit_error": True,
"object_store_memory": object_store_memory,
"address": redis_address,
"_redis_password": redis_password,
"_memory": object_store_memory,
}
ray.init(**ray_init_kwargs)
if Backend.get() == "Cudf":
from modin.engines.ray.cudf_on_ray.frame.gpu_manager import GPUManager
from modin.engines.ray.cudf_on_ray.frame.partition_manager import (
GPU_MANAGERS, )
# Check that GPU_MANAGERS is empty because _update_engine can be called multiple times
if not GPU_MANAGERS:
for i in range(GpuCount.get()):
GPU_MANAGERS.append(GPUManager.remote(i))
_move_stdlib_ahead_of_site_packages()
ray.worker.global_worker.run_function_on_all_workers(
_move_stdlib_ahead_of_site_packages)
ray.worker.global_worker.run_function_on_all_workers(_import_pandas)
num_cpus = int(ray.cluster_resources()["CPU"])
num_gpus = int(ray.cluster_resources().get("GPU", 0))
if Backend.get() == "Cudf":
NPartitions._put(num_gpus)
else:
NPartitions._put(num_cpus)
def get_max_filename_length():
fd = os.open(".", os.O_RDONLY)
info = os.fstatvfs(fd)
return info.f_namemax
# 获取元组 info = os.fstat(fd) print "文件信息 :", info # 获取文件 uid print "文件 UID :%d" % info.st_uid # 获取文件 gid print "文件 GID :%d" % info.st_gid # 关闭文件 os.close(fd) # 18 os.fstatvfs(fd) # 返回包含文件描述符fd的文件的文件系统的信息，像 statvfs() Unix上可用。 # 打开文件 fd = os.open( "foo.txt", os.O_RDWR|os.O_CREAT ) # 获取元组 info = os.fstatvfs(fd) print "文件信息 :", info # 获取文件名最大长度 print "文件名最大长度 :%d" % info.f_namemax # 获取可用块数 print "可用块数 :%d" % info.f_bfree # 关闭文件 os.close(fd) # 19 os.fsync(fd) # 强制将文件描述符为fd的文件写入硬盘。 # 打开文件 fd = os.open( "foo.txt", os.O_RDWR|os.O_CREAT ) # 写入字符串 os.write(fd, "This is test") # 使用 fsync() 方法.
def get_max_filename_length():
fd = os.open(".", os.O_RDONLY)
info = os.fstatvfs(fd)
return info.f_namemax
def run(self):
if self.run_locally == 1:
return self.work()
else:
# Set up temp folder in shared directory (trim to max filename length)
base_tmp_dir = self.shared_tmp_dir
random_id = '%016x' % random.getrandbits(64)
folder_name = self.task_id + '-' + random_id
self.tmp_dir = os.path.join(base_tmp_dir, folder_name)
max_filename_length = os.fstatvfs(0).f_namemax
self.tmp_dir = self.tmp_dir[:max_filename_length]
logger.info("Tmp dir: %s", self.tmp_dir)
to_copy = [d for d in os.listdir() if d != ".git"]
if not os.path.exists(self.tmp_dir):
os.makedirs(self.tmp_dir)
for f in to_copy:
if os.path.isfile(f):
copyfile(f, os.path.join(self.tmp_dir, f))
else:
copytree(f, os.path.join(self.tmp_dir, f))
# Dump the code to be run into a pickle file
logging.debug("Dumping pickled class")
self._dump(self.tmp_dir)
if not self.no_tarball:
# Make sure that all the class's dependencies are tarred and available
# This is not necessary if luigi is importable from the cluster node
logging.debug("Tarballing dependencies")
# Grab luigi and the module containing the code to be run
packages = [luigi] + [
__import__(self.__module__, None, None, 'dummy')
]
create_packages_archive(
packages, os.path.join(self.tmp_dir, "packages.tar"))
# make a stamp indicator in the folder
# generate unique descriptive stamp for current commit
get_commit_property = lambda s: subprocess.check_output(
"git --no-pager log -1 --format=%{}".format(
s), shell=True).decode("utf-8").strip()
commit_time = get_commit_property("ci")[:16] \
.replace(" ", "-") \
.replace(":", "-")
commit_author = get_commit_property("ce").split("@")[0]
commit_hash = get_commit_property("h")
stamp = "{}-{}-{}".format(commit_time, commit_author, commit_hash)
# write out stamp to temp folder
with open(os.path.join(self.tmp_dir, "stamp"), "w") as stamp_file:
stamp_file.write(stamp)
# Build a qsub argument that will run sge_runner.py on the directory
# we've specified
runner_path = os.path.join("utility", "sge_runner.py")
if runner_path.endswith("pyc"):
runner_path = runner_path[:-3] + "py"
# enclose tmp_dir in quotes to protect from special escape chars
job_str = 'python {0} "{1}" "{2}"'.format(runner_path,
self.tmp_dir,
os.getcwd())
if self.no_tarball:
job_str += ' "--no-tarball"'
qsub_template = """echo {cmd} | {qsub_command} -V -r y -pe {pe} {n_cpu} -N {job_name} -l m_mem_free={mem_free} -sync y"""
submit_cmd = qsub_template.format(cmd=job_str,
job_name=self.job_name,
pe=self.parallel_env,
n_cpu=self.n_cpu,
mem_free=self.mem_free,
qsub_command=self.qsub_command)
logger.debug('qsub command: \n' + submit_cmd)
# Submit the job and grab job ID
try:
output = subprocess.check_output(submit_cmd,
shell=True,
stderr=subprocess.STDOUT)
logger.debug("qsub job complete with output:\n" +
output.decode('utf-8'))
except subprocess.CalledProcessError as e:
logger.error("qsub submission failed with output:\n" +
e.output.decode('utf-8'))
if os.path.exists(os.path.join(self.tmp_dir, "job.err")):
with open(os.path.join(self.tmp_dir, "job.err"),
"r") as err:
logger.error(err.read())
# wait a beat, to give things a chance to settle
time.sleep(2)
# check whether the file exists
if not self.output().exists():
raise Exception("qsub failed to produce output")
else:
# delete the temporaries, if they're there.
if self.tmp_dir and os.path.exists(self.tmp_dir):
logger.info('Removing temporary directory %s',
self.tmp_dir)
subprocess.call(["rm", "-rf", self.tmp_dir])
def test_fstatvfs(self):
try:
os.fstatvfs(0)
except OSError, e:
py.test.skip("the underlying os.fstatvfs() failed: %s" % e)
def view(self, user, suite, path, path_in_tar=None, mode=None):
"""View a text log file."""
f_name = self._get_user_suite_dir(user, suite, path)
conf = ResourceLocator.default().get_conf()
view_size_max = int(conf.get_value(
["rose-bush", "view-size-max"], self.VIEW_SIZE_MAX))
if path_in_tar:
tar_f = tarfile.open(f_name, 'r:gz')
try:
tar_info = tar_f.getmember(path_in_tar)
except KeyError:
raise cherrypy.HTTPError(404)
f_size = tar_info.size
f = tar_f.extractfile(path_in_tar)
if f.read(2) == "#!":
mime = MIME_TEXT_PLAIN
else:
mime = mimetypes.guess_type(
urllib.pathname2url(path_in_tar))[0]
f.seek(0)
if (mode == "download" or
f_size > view_size_max or
mime and
(not mime.startswith("text/") or mime.endswith("html"))):
t = NamedTemporaryFile()
f_bsize = os.fstatvfs(t.fileno()).f_bsize
while True:
bytes = f.read(f_bsize)
if not bytes:
break
t.write(bytes)
cherrypy.response.headers["Content-Type"] = mime
try:
return cherrypy.lib.static.serve_file(t.name, mime)
finally:
t.close()
s = f.read()
f.close()
else:
f_size = os.stat(f_name).st_size
if open(f_name).read(2) == "#!":
mime = MIME_TEXT_PLAIN
else:
mime = mimetypes.guess_type(urllib.pathname2url(f_name))[0]
if (mode == "download" or
f_size > view_size_max or
mime and
(not mime.startswith("text/") or mime.endswith("html"))):
cherrypy.response.headers["Content-Type"] = mime
return cherrypy.lib.static.serve_file(f_name, mime)
s = open(f_name).read()
try:
if mode in [None, "text"]:
s = jinja2.escape(s)
lines = [unicode(line) for line in s.splitlines()]
except UnicodeDecodeError:
return cherrypy.lib.static.serve_file(f_name, MIME_TEXT_PLAIN)
name = path
if path_in_tar:
name = path_in_tar
job_entry = None
if name.startswith("log/job/"):
names = self.suite_engine_proc.parse_job_log_rel_path(name)
if len(names) == 4:
cycle, task, submit_num, ext = names
entries = self.suite_engine_proc.get_suite_job_events(
user, suite, [cycle], [task], None, None, None, None)[0]
for entry in entries:
if entry["submit_num"] == int(submit_num):
job_entry = entry
break
if fnmatch(os.path.basename(path), "rose*.conf"):
file_content = "rose-conf"
else:
file_content = self.suite_engine_proc.is_conf(path)
template = self.template_env.get_template("view.html")
data = {}
data.update(self._get_suite_logs_info(user, suite))
return template.render(
rose_version=self.rose_version,
script=cherrypy.request.script_name,
method="view",
time=strftime("%Y-%m-%dT%H:%M:%S+0000", gmtime()),
logo=self.logo,
title=self.title,
host=self.host_name,
user=user,
suite=suite,
path=path,
path_in_tar=path_in_tar,
f_name=f_name,
mode=mode,
file_content=file_content,
lines=lines,
entry=job_entry,
**data)
def fstatvfs(self, handle):
fo = self._files.get(handle, None)
if fo:
return os.fstatvfs(fo.fileno())
os.chown(path, uid, gid) # 更改文件所有者 os.chroot(path) # 改变当前进程的根目录 os.close(fd) # 关闭文件描述符 fd os.closerange(fd_low, fd_high) # 关闭所有文件描述符，从 fd_low (包含) 到 fd_high (不包含), 错误会忽略 os.curdir # 返回当前目录：（'.'） os.dup(fd) # 复制文件描述符 fd os.dup2(fd, fd2) # 将一个文件描述符 fd 复制到另一个 fd2 os.environ # 获取系统环境变量 os.fchdir(fd) # 通过文件描述符改变当前工作目录 os.fchmod(fd, mode) # 改变一个文件的访问权限，该文件由参数fd指定，参数mode是Unix下的文件访问权限。 os.fchown(fd, uid, gid) # 修改一个文件的所有权，这个函数修改一个文件的用户ID和用户组ID，该文件由文件描述符fd指定。 os.fdatasync(fd) # 强制将文件写入磁盘，该文件由文件描述符fd指定，但是不强制更新文件的状态信息。 os.fdopen(fd[, mode[, bufsize]]) # 通过文件描述符 fd 创建一个文件对象，并返回这个文件对象 os.fpathconf(fd, name) # 返回一个打开的文件的系统配置信息。name为检索的系统配置的值，它也许是一个定义系统值的字符串，这些名字在很多标准中指定（POSIX.1, Unix 95, Unix 98, 和其它）。 os.fstat(fd) # 返回文件描述符fd的状态，像stat()。 os.fstatvfs(fd) # 返回包含文件描述符fd的文件的文件系统的信息，像 statvfs() os.fsync(fd) # 强制将文件描述符为fd的文件写入硬盘。 os.ftruncate(fd, length) # 裁剪文件描述符fd对应的文件, 所以它最大不能超过文件大小。 os.getcwd() # 返回当前工作目录 os.getcwdu() # 返回一个当前工作目录的Unicode对象 os.isatty(fd) # 如果文件描述符fd是打开的，同时与tty(-like)设备相连，则返回true, 否则False。 os.lchflags(path, flags) # 设置路径的标记为数字标记，类似 chflags()，但是没有软链接 os.lchmod(path, mode) # 修改连接文件权限 os.lchown(path, uid, gid) # 更改文件所有者，类似 chown，但是不追踪链接。 os.link(src, dst) # 创建硬链接，名为参数 dst，指向参数 src os.listdir(path) # 返回path指定的文件夹包含的文件或文件夹的名字的列表。 os.lseek(fd, pos, how) # 设置文件描述符 fd当前位置为pos, how方式修改: SEEK_SET 或者 0 设置从文件开始的计算的pos; SEEK_CUR或者 1 则从当前位置计算; os.SEEK_END或者2则从文件尾部开始. 在unix，Windows中有效 os.lstat(path) # 像stat(),但是没有软链接 os.linesep # 当前平台使用的行终止符，win下为"\t\n",Linux下为"\n" os.major(device) # 从原始的设备号中提取设备major号码 (使用stat中的st_dev或者st_rdev field)。 os.makedev(major, minor) # 以major和minor设备号组成一个原始设备号