def truncate(self, filepath, size):
with self._get_lock():
ascii_path = filepath.encode('ascii', 'ignore')
localfs_path = self._make_localfs_path(ascii_path)
fd = os.open(localfs_path, os.O_WRONLY | os.O_CREAT)
os.ftruncate(fd, size)
os.close(fd)
def prepare_rootfs_ext(self, rootfs, oe_builddir, rootfs_dir,
native_sysroot, pseudo):
"""
Prepare content for an ext2/3/4 rootfs partition.
"""
du_cmd = "du -ks %s" % rootfs_dir
out = exec_cmd(du_cmd)
actual_rootfs_size = int(out.split()[0])
rootfs_size = self.get_rootfs_size(actual_rootfs_size)
with open(rootfs, 'w') as sparse:
os.ftruncate(sparse.fileno(), rootfs_size * 1024)
extraopts = self.mkfs_extraopts or "-F -i 8192"
label_str = ""
if self.label:
label_str = "-L %s" % self.label
mkfs_cmd = "mkfs.%s %s %s %s -d %s" % \
(self.fstype, extraopts, rootfs, label_str, rootfs_dir)
exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
mkfs_cmd = "fsck.%s -pvfD %s" % (self.fstype, rootfs)
exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
def __init__(self, pid_path):
self._pid_path = pid_path
self._other_running = False
ensuredirs(self._pid_path)
self._lockfile = None
try:
self._lockfile = os.open(self._pid_path, os.O_CREAT | os.O_WRONLY)
except:
raise SoleError('Cannot open lockfile (path = %s)' % self._pid_path)
try:
fcntl.lockf(self._lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
self._other_running = True
try:
f = open(self._pid_path, 'r')
pid = f.read().strip()
f.close()
except:
pid = '?'
raise SoleError('Other instance is running (pid = %s)' % pid)
try:
os.ftruncate(self._lockfile, 0)
os.write(self._lockfile, '%i\n' % os.getpid())
os.fsync(self._lockfile)
except:
pass # the pid is only stored for user information, so this is allowed to fail
def truncate_and_unlink(path, increment=256*1024**2, delay=0.2):
"""Truncate a file to zero bytes before unlinking
Truncation is done in ``increment`` bytes with a sleep delay
of ``delay`` seconds between each truncation step.
Once the file is zero bytes in size it will be removed/unlinked
from the filesystem.
:raises: OSError on error
"""
LOG.debug("truncate_and_unlink(path=%r, increment=%r, delay=%.2fs)",
path, format_bytes(increment), delay)
fd = os.open(path, os.O_RDWR)
size = os.fstat(fd).st_size
while size > 0:
start = time.time()
length = size - increment
if length < 0:
length = 0
os.ftruncate(fd, length)
LOG.debug("truncate(%s, length=%s) in %.2fs",
path, format_bytes(length), time.time() - start)
time.sleep(delay)
size = os.fstat(fd).st_size
os.unlink(path)
LOG.debug("unlink(%s)", path)
def _general_print(head, color, msg=None, stream=None, level="normal"):
global LOG_CONTENT
if not stream:
stream = sys.stdout
if LOG_LEVELS[level] > LOG_LEVEL:
# skip
return
# encode raw 'unicode' str to utf8 encoded str
if msg and isinstance(msg, unicode):
msg = msg.encode("utf-8", "ignore")
errormsg = ""
if CATCHERR_BUFFILE_FD > 0:
size = os.lseek(CATCHERR_BUFFILE_FD, 0, os.SEEK_END)
os.lseek(CATCHERR_BUFFILE_FD, 0, os.SEEK_SET)
errormsg = os.read(CATCHERR_BUFFILE_FD, size)
os.ftruncate(CATCHERR_BUFFILE_FD, 0)
# append error msg to LOG
if errormsg:
LOG_CONTENT += errormsg
# append normal msg to LOG
save_msg = msg.strip() if msg else None
if save_msg:
global HOST_TIMEZONE
timestr = time.strftime("[%m/%d %H:%M:%S] ", time.gmtime(time.time() - HOST_TIMEZONE))
LOG_CONTENT += timestr + save_msg + "\n"
if errormsg:
_color_print("", NO_COLOR, errormsg, stream, level)
_color_print(head, color, msg, stream, level)
def expand(self, create = False, size = None):
flags = os.O_WRONLY
if create:
flags |= os.O_CREAT
if not os.path.exists(self.lofile):
makedirs(os.path.dirname(self.lofile))
if size is None:
size = self.size
msger.debug("Extending sparse file %s to %d" % (self.lofile, size))
if create:
fd = os.open(self.lofile, flags, 0644)
else:
fd = os.open(self.lofile, flags)
if size <= 0:
size = 1
try:
os.ftruncate(fd, size)
except:
# may be limited by 2G in 32bit env
os.ftruncate(fd, 2**31L)
os.close(fd)
def _commit_puts(self, item_list=None):
"""
Scan for .pending files and commit the found records by feeding them
to merge_items(). Assume that lock_parent_directory has already been
called.
:param item_list: A list of items to commit in addition to .pending
"""
if self.db_file == ':memory:' or not os.path.exists(self.pending_file):
return
if item_list is None:
item_list = []
self._preallocate()
if not os.path.getsize(self.pending_file):
if item_list:
self.merge_items(item_list)
return
with open(self.pending_file, 'r+b') as fp:
for entry in fp.read().split(':'):
if entry:
try:
self._commit_puts_load(item_list, entry)
except Exception:
self.logger.exception(
_('Invalid pending entry %(file)s: %(entry)s'),
{'file': self.pending_file, 'entry': entry})
if item_list:
self.merge_items(item_list)
try:
os.ftruncate(fp.fileno(), 0)
except OSError as err:
if err.errno != errno.ENOENT:
raise
def count(delta, file_path):
"""
Increments counter file and returns the max number of times the file
has been modified. Counter data must be in the form:
concurrent tasks, max concurrent tasks (counter should be initialized to 0,0)
:param int delta: increment value
:param str file_path: path to shared counter file
:return int max concurrent tasks:
"""
fd = os.open(file_path, os.O_RDWR)
try:
fcntl.flock(fd, fcntl.LOCK_EX)
try:
s = os.read(fd, 10)
value, maxValue = map(int, s.split(','))
value += delta
if value > maxValue: maxValue = value
os.lseek(fd, 0, 0)
os.ftruncate(fd, 0)
os.write(fd, ','.join(map(str, (value, maxValue))))
finally:
fcntl.flock(fd, fcntl.LOCK_UN)
finally:
os.close(fd)
return maxValue
def _create_local_file(self, progresscb, size_bytes):
"""
Helper function which attempts to build self.path
"""
fd = None
path = self._path
sparse = self._sparse
try:
try:
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_DSYNC)
if sparse:
os.ftruncate(fd, size_bytes)
else:
# 1 meg of nulls
mb = 1024 * 1024
buf = '\x00' * mb
left = size_bytes
while left > 0:
if left < mb:
buf = '\x00' * left
left = max(left - mb, 0)
os.write(fd, buf)
progresscb.update(size_bytes - left)
except OSError, e:
raise RuntimeError(_("Error creating diskimage %s: %s") %
(path, str(e)))
finally:
if fd is not None:
os.close(fd)
progresscb.end(size_bytes)
def write_pid_file(pid_file, pid):
import fcntl
import stat
try:
fd = os.open(pid_file, os.O_RDWR | os.O_CREAT,
stat.S_IRUSR | stat.S_IWUSR)
except OSError as e:
comm.print_exception(e)
return -1
flags = fcntl.fcntl(fd, fcntl.F_GETFD)
assert flags != -1
flags |= fcntl.FD_CLOEXEC
r = fcntl.fcntl(fd, fcntl.F_SETFD, flags)
assert r != -1
# There is no platform independent way to implement fcntl(fd, F_SETLK, &fl)
# via fcntl.fcntl. So use lockf instead
try:
fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB, 0, 0, os.SEEK_SET)
except IOError:
r = os.read(fd, 32)
if r:
logging.error('already started at pid %s' % comm.to_str(r))
else:
logging.error('already started')
os.close(fd)
return -1
os.ftruncate(fd, 0)
os.write(fd, comm.to_bytes(str(pid)))
return 0
def updatemanifest(manifestfile, versionname, versioncode):
dom1 = parse(manifestfile)
dom1.documentElement.setAttribute("android:versionName", versionname)
dom1.documentElement.setAttribute("android:versionCode", versioncode)
f = os.open(manifestfile, os.O_RDWR)
os.ftruncate(f,0);
os.write(f, dom1.toxml())
def _read_write(self, data):
total_bytes = reduce(operator.add, map(len, data))
n = coro.many_lio_writes(self.fd, 0, data)
self.assertEqual(n, total_bytes)
a = coro.many_lio_reads(self.fd, 0, total_bytes)
self.assertEqual(a, data)
os.ftruncate(self.fd, 0)
def download_to_vpool(url, path, overwrite_if_exists=False):
"""
Special method to download to vpool because voldrv does not support extending file at write
:param url: URL to download from
:type url: str
:param path: Path to download to
:type path: str
:param overwrite_if_exists: Overwrite if file already exists
:type overwrite_if_exists: bool
:return: None
"""
print url
print path
if os.path.exists(path) and not overwrite_if_exists:
return
u = urllib.urlopen(url)
file_size = u.info()['Content-Length']
bsize = 4096 * 1024
VDiskController.create_volume(path, 0)
with open(path, "wb") as f:
size_written = 0
os.ftruncate(f.fileno(), int(file_size))
while 1:
s = u.read(bsize)
size_written += len(s)
f.write(s)
if len(s) < bsize:
break
u.close()
def _general_print(head, color, msg=None, stream=None, level='normal'):
global LOG_CONTENT
if not stream:
stream = sys.stdout
if LOG_LEVELS[level] > LOG_LEVEL:
# skip
return
# encode raw 'unicode' str to utf8 encoded str
if msg and isinstance(msg, unicode):
msg = msg.encode('utf-8', 'ignore')
errormsg = ''
if CATCHERR_BUFFILE_FD > 0:
size = os.lseek(CATCHERR_BUFFILE_FD , 0, os.SEEK_END)
os.lseek(CATCHERR_BUFFILE_FD, 0, os.SEEK_SET)
errormsg = os.read(CATCHERR_BUFFILE_FD, size)
os.ftruncate(CATCHERR_BUFFILE_FD, 0)
# append error msg to LOG
if errormsg:
LOG_CONTENT += errormsg
# append normal msg to LOG
save_msg = msg.strip() if msg else None
if save_msg:
timestr = time.strftime("[%m/%d %H:%M:%S %Z] ", time.localtime())
LOG_CONTENT += timestr + save_msg + '\n'
if errormsg:
_color_print('', NO_COLOR, errormsg, stream, level)
_color_print(head, color, msg, stream, level)
def truncateFile(self, fusepath, size):
'''
Unconditionally truncate the file. Don't check to see if the user has
access.
'''
try:
self.lockFile(fusepath)
opcodes = self._genCacheOpcodes(fusepath)
self._validateCache(fusepath, opcodes)
realpath = self._generatePath(fusepath, opcodes)
logger.debug('Truncating %s to %d bytes.' % (realpath, size))
fd = os.open(realpath, os.O_RDWR)
os.ftruncate(fd, size)
os.close(fd)
# Since we wrote to the file, invalidate the stat cache if it exists.
self._invalidateStatCache(realpath)
return 0
finally:
self.unlockFile(fusepath)
def with_pid_file(pid_file, pid):
""""""
try:
fd = os.open(pid_file, os.O_RDWR | os.O_CREAT, stat.S_IRUSR | stat.S_IWUSR)
except OSError as e:
logging.error(e.message)
return False
flags = fcntl.fcntl(fd, fcntl.F_GETFD)
assert flags != -1
flags = fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)
assert flags != -1
try:
fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
pid = os.read(fd, 32)
logging.error('already started %(pid)s' % {'pid': pid})
os.close(fd)
return False
os.ftruncate(fd, 0)
os.write(fd, str(pid))
return True
def _create_file(self, filename, size, sparse=True):
try:
f = open(filename, "w+")
except (OSError, IOError):
raise ExecutionError("Could not open %s" % filename)
try:
if sparse:
try:
os.posix_fallocate(f.fileno(), 0, size)
except AttributeError:
# Prior to version 3.3, Python does not provide fallocate
os.ftruncate(f.fileno(), size)
else:
self.shell.log.info("Writing %d bytes" % size)
while size > 0:
write_size = min(size, 1024)
f.write("\0" * write_size)
size -= write_size
except (OSError, IOError):
os.remove(filename)
raise ExecutionError("Could not expand file to %d bytes" % size)
except OverflowError:
raise ExecutionError("The file size is too large (%d bytes)" % size)
finally:
f.close()
def forward_outputs(outs):
for fd in (1, 2):
if fd in outs:
os.lseek(fd, 0, os.SEEK_SET)
with io.open(fd, closefd=False) as f:
STD_STREAM[fd].writelines(f)
os.ftruncate(fd, 0)
def create_anon_file(sz):
fd, fn = tempfile.mkstemp(dir=XDG_RUNTIME_DIR) # create temporary file
flags = fcntl.fcntl(fd, fcntl.F_GETFD)
fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)
os.unlink(fn) # make it anonymous
os.ftruncate(fd, sz) # limit the size
return fd
def _init_mmap(path=None, filename=None, size=PAGESIZE):
"""Given path, filename => filename, size, mmap"""
if path is None:
path = DEFAULT_PATH
if filename is None:
filename = 'mmstats-%d' % os.getpid()
tid = libgettid.gettid()
if tid:
filename += '-%d' % tid
full_path = os.path.join(path, filename)
# Create new empty file to back memory map on disk
fd = os.open(full_path, os.O_CREAT | os.O_TRUNC | os.O_RDWR)
if size > PAGESIZE:
if size % PAGESIZE:
size = size + (PAGESIZE - (size % PAGESIZE))
else:
size = PAGESIZE
# Zero out the file
os.ftruncate(fd, size)
m = mmap.mmap(fd, size, mmap.MAP_SHARED, mmap.PROT_WRITE)
return (full_path, size, m)
def __init__(self, size, fd=-1):
self.size = size
self.fd = fd
if fd == -1:
if PY3:
self.fd, name = tempfile.mkstemp(
prefix='pym-%d-' % (os.getpid(),),
dir=util.get_temp_dir(),
)
os.unlink(name)
util.Finalize(self, os.close, (self.fd,))
with io.open(self.fd, 'wb', closefd=False) as f:
bs = 1024 * 1024
if size >= bs:
zeros = b'\0' * bs
for _ in range(size // bs):
f.write(zeros)
del(zeros)
f.write(b'\0' * (size % bs))
assert f.tell() == size
else:
name = tempfile.mktemp(
prefix='pym-%d-' % (os.getpid(),),
dir=util.get_temp_dir(),
)
self.fd = os.open(
name, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600,
)
util.Finalize(self, os.close, (self.fd,))
os.unlink(name)
os.ftruncate(self.fd, size)
self.buffer = mmap.mmap(self.fd, self.size)
def _commit_puts(self, item_list=None):
"""
Scan for .pending files and commit the found records by feeding them
to merge_items().
:param item_list: A list of items to commit in addition to .pending
"""
if self.db_file == ":memory:" or not os.path.exists(self.pending_file):
return
if item_list is None:
item_list = []
with lock_parent_directory(self.pending_file, self.pending_timeout):
self._preallocate()
if not os.path.getsize(self.pending_file):
if item_list:
self.merge_items(item_list)
return
with open(self.pending_file, "r+b") as fp:
for entry in fp.read().split(":"):
if entry:
try:
self._commit_puts_load(item_list, entry)
except Exception:
self.logger.exception(
_("Invalid pending entry %(file)s: %(entry)s"),
{"file": self.pending_file, "entry": entry},
)
if item_list:
self.merge_items(item_list)
try:
os.ftruncate(fp.fileno(), 0)
except OSError as err:
if err.errno != errno.ENOENT:
raise
def __init__(self, u_boot_console):
"""Initialize a new GptTestDiskImage object.
Args:
u_boot_console: A U-Boot console.
Returns:
Nothing.
"""
filename = 'test_gpt_disk_image.bin'
persistent = u_boot_console.config.persistent_data_dir + '/' + filename
self.path = u_boot_console.config.result_dir + '/' + filename
if os.path.exists(persistent):
u_boot_console.log.action('Disk image file ' + persistent +
' already exists')
else:
u_boot_console.log.action('Generating ' + persistent)
fd = os.open(persistent, os.O_RDWR | os.O_CREAT)
os.ftruncate(fd, 4194304)
os.close(fd)
cmd = ('sgdisk', '-U', '375a56f7-d6c9-4e81-b5f0-09d41ca89efe',
persistent)
u_boot_utils.run_and_log(u_boot_console, cmd)
cmd = ('sgdisk', '--new=1:2048:2560', '-c 1:part1', persistent)
u_boot_utils.run_and_log(u_boot_console, cmd)
cmd = ('sgdisk', '--new=2:4096:4608', '-c 2:part2', persistent)
u_boot_utils.run_and_log(u_boot_console, cmd)
cmd = ('sgdisk', '-l', persistent)
u_boot_utils.run_and_log(u_boot_console, cmd)
cmd = ('cp', persistent, self.path)
u_boot_utils.run_and_log(u_boot_console, cmd)
def tidy(self):
'''
Remove dead space.
'''
INTSIZE = (self.xwidth - self.width) // 2
entries = []
data = []
# Fetch content, and update pointers without saving to the file yet.
ptr = self.offset + self.size * self.xwidth
for i in range(self.items):
entry = self.__read(self.xwidth - self.width, self.offset + i * self.xwidth + self.width)
entry = entry.decode('utf-8', 'strict')
offset = int(entry[:len(entry) // 2])
length = int(entry[len(entry) // 2:])
if length > 0:
data.append(self.__read(length, offset))
entries.append((ptr if length > 0 else 0, length))
ptr += length
# Update stored content.
ptr = self.offset + self.size * self.xwidth
for datum in range(data):
entry = self.__write(datum, ptr)
ptr += len(datum)
os.ftruncate(self.fd, ptr)
# Update all pointers in the entry list.
for (ptr, length) in range(self.items):
entry = ('%0*i%0*i' % (INTSIZE, ptr, INTSIZE, length)).encode('utf-8')
self.__write(entry, self.offset + i * self.xwidth + self.width)
def modify(guid, worker, filename, key):
obj = dict(guid=guid, total=0, workers=[], extraTotal=0, extraWorkers=[])
fd = os.open(filename, os.O_CREAT|os.O_RDWR)
raw = os.read(fd, 1000000)
if (raw is None) or ('' == raw):
os.ftruncate(fd, 0)
else:
tmp = json.loads(raw)
if ('guid' not in tmp) or (guid != tmp['guid']):
os.ftruncate(fd, 0)
else:
obj = tmp
if 'main' == key:
obj['total'] += 1
obj['workers'].append(worker)
else:
obj['extraTotal'] += 1
obj['extraWorkers'].append(worker)
os.lseek(fd, 0, os.SEEK_SET)
os.write(fd, json.dumps(obj, indent=4, sort_keys=True))
def __lock_pidfile(filename, verbose=True):
fd = os.open(filename, os.O_CREAT|os.O_RDWR)
try:
pid = 0
# Lock for read, to read PID
fcntl.flock(fd, fcntl.LOCK_SH)
pid = 0
str_pid = os.read(fd, 100)
try:
pid = int(str_pid)
except ValueError:
# XXX bad pidfile - clean trash
os.ftruncate(fd, 0)
if verbose: sys.stderr.write("Bad Pidfile!\nPidfile contains trash: \"%s\"\nPidfile nas been cleaned!\n" % str_pid)
# reLock for write or exit
fcntl.flock(fd, fcntl.LOCK_EX|fcntl.LOCK_NB)
# save PID
process_id = os.getpid()
os.ftruncate(fd, 0)
os.lseek(fd, 0, os.SEEK_SET)
os.write(fd, str(process_id).encode('utf-8'))
os.fsync(fd)
# Re-lock for read to make the PID accessible
fcntl.flock(fd, fcntl.LOCK_SH)
return fd, pid
except IOError:
os.close(fd)
return None, pid
def _general_print(head, color, msg = None, stream = None, level = 'normal'):
global LOG_CONTENT
if not stream:
stream = sys.stdout
if LOG_LEVELS[level] > LOG_LEVEL:
# skip
return
errormsg = ''
if CATCHERR_BUFFILE_FD > 0:
size = os.lseek(CATCHERR_BUFFILE_FD , 0, os.SEEK_END)
os.lseek(CATCHERR_BUFFILE_FD, 0, os.SEEK_SET)
errormsg = os.read(CATCHERR_BUFFILE_FD, size)
os.ftruncate(CATCHERR_BUFFILE_FD, 0)
if LOG_FILE_FP:
if errormsg:
LOG_CONTENT += errormsg
save_msg = msg.strip() if msg else None
if save_msg:
global HOST_TIMEZONE
timestr = time.strftime("[%m/%d %H:%M:%S] ",
time.gmtime(time.time() - HOST_TIMEZONE))
LOG_CONTENT += timestr + save_msg + '\n'
if errormsg:
_color_print('', NO_COLOR, errormsg, stream, level)
_color_print(head, color, msg, stream, level)
def prepare_rootfs_btrfs(self, rootfs, oe_builddir, rootfs_dir,
native_sysroot, pseudo):
"""
Prepare content for a btrfs rootfs partition.
Currently handles ext2/3/4 and btrfs.
"""
du_cmd = "du -ks %s" % rootfs_dir
out = exec_cmd(du_cmd)
actual_rootfs_size = int(out.split()[0])
extra_blocks = self.get_extra_block_count(actual_rootfs_size)
if extra_blocks < self.extra_space:
extra_blocks = self.extra_space
rootfs_size = actual_rootfs_size + extra_blocks
rootfs_size *= self.overhead_factor
msger.debug("Added %d extra blocks to %s to get to %d total blocks" % \
(extra_blocks, self.mountpoint, rootfs_size))
with open(rootfs, 'w') as sparse:
os.ftruncate(sparse.fileno(), rootfs_size * 1024)
label_str = ""
if self.label:
label_str = "-L %s" % self.label
mkfs_cmd = "mkfs.%s -b %d -r %s %s %s" % \
(self.fstype, rootfs_size * 1024, rootfs_dir, label_str, rootfs)
exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
def release(self, fd, dest, fsize, filename, created):
"""release handler.
@param fd file discripter
@param writelen size of data to be written
"""
os.lseek(fd, 0, os.SEEK_SET)
DRDFSLog.debug("fd = " + str(fd))
if created == False:
self.repl_info.ReplInfoWhenClose(filename, dest,
self.access_info)
else:
try:
buf = os.read(fd, conf.bufsize)
except os.error, e:
print "OSError in release (%s)" %(e)
l = buf.rsplit(',')
size = string.atol(l[2])
try:
buf = l[0] + ',' + l[1] + ',' + str(fsize) + ',\n'
DRDFSLog.debug("write to meta file %s" % buf)
os.ftruncate(fd, len(buf))
os.lseek(fd, 0, os.SEEK_SET)
os.write(fd, buf)
os.fsync(fd)
except os.error, e:
print "OSError in release (%s)" %(e)
def _general_print(head, color, msg=None, stream=None, level="normal"):
global LOG_CONTENT
if not stream:
stream = sys.stdout
if LOG_LEVELS[level] > LOG_LEVEL:
# skip
return
errormsg = ""
if CATCHERR_BUFFILE_FD > 0:
size = os.lseek(CATCHERR_BUFFILE_FD, 0, os.SEEK_END)
os.lseek(CATCHERR_BUFFILE_FD, 0, os.SEEK_SET)
errormsg = os.read(CATCHERR_BUFFILE_FD, size)
os.ftruncate(CATCHERR_BUFFILE_FD, 0)
if LOG_FILE_FP:
if errormsg:
LOG_CONTENT += errormsg
save_msg = msg.strip()
if save_msg:
timestr = datetime.now().strftime("[%m/%d %H:%M:%S] ")
LOG_CONTENT += timestr + save_msg + "\n"
if errormsg:
_color_print("", NO_COLOR, errormsg, stream, level)
_color_print(head, color, msg, stream, level)
def _clone_local(self, meter, size_bytes):
if self._input_path == "/dev/null":
# Not really sure why this check is here,
# but keeping for compat
logging.debug("Source dev was /dev/null. Skipping")
return
if self._input_path == self._output_path:
logging.debug("Source and destination are the same. Skipping.")
return
# If a destination file exists and sparse flag is True,
# this priority takes an existing file.
if (not os.path.exists(self._output_path) and self._sparse):
clone_block_size = 4096
sparse = True
fd = None
try:
fd = os.open(self._output_path, os.O_WRONLY | os.O_CREAT, 0640)
os.ftruncate(fd, size_bytes)
finally:
if fd:
os.close(fd)
else:
clone_block_size = 1024 * 1024 * 10
sparse = False
logging.debug("Local Cloning %s to %s, sparse=%s, block_size=%s",
self._input_path, self._output_path, sparse,
clone_block_size)
zeros = '\0' * 4096
src_fd, dst_fd = None, None
try:
try:
src_fd = os.open(self._input_path, os.O_RDONLY)
dst_fd = os.open(self._output_path, os.O_WRONLY | os.O_CREAT,
0640)
i = 0
while 1:
l = os.read(src_fd, clone_block_size)
s = len(l)
if s == 0:
meter.end(size_bytes)
break
# check sequence of zeros
if sparse and zeros == l:
os.lseek(dst_fd, s, 1)
else:
b = os.write(dst_fd, l)
if s != b:
meter.end(i)
break
i += s
if i < size_bytes:
meter.update(i)
except OSError, e:
raise RuntimeError(
_("Error cloning diskimage %s to %s: %s") %
(self._input_path, self._output_path, str(e)))
finally:
if src_fd is not None:
os.close(src_fd)
if dst_fd is not None:
os.close(dst_fd)
def f(fi, len):
os.ftruncate(fi, len)
def copy(self, sync=True, verify=True):
"""
Copy the image to the destination file using bmap. The 'sync' argument
defines whether the destination file has to be synchronized upon
return. The 'verify' argument defines whether the checksum has to be
verified while copying.
"""
# Create the queue for block batches and start the reader thread, which
# will read the image in batches and put the results to '_batch_queue'.
self._batch_queue = Queue.Queue(self._batch_queue_len)
thread.start_new_thread(self._get_data, (verify, ))
blocks_written = 0
bytes_written = 0
fsync_last = 0
self._progress_started = False
self._progress_index = 0
self._progress_time = datetime.datetime.now()
# Read the image in '_batch_blocks' chunks and write them to the
# destination file
while True:
batch = self._batch_queue.get()
if batch is None:
# No more data, the image is written
break
elif batch[0] == "error":
# The reader thread encountered an error and passed us the
# exception.
exc_info = batch[1]
raise exc_info[0], exc_info[1], exc_info[2]
(start, end, buf) = batch[1:4]
assert len(buf) <= (end - start + 1) * self.block_size
assert len(buf) > (end - start) * self.block_size
self._f_dest.seek(start * self.block_size)
# Synchronize the destination file if we reached the watermark
if self._dest_fsync_watermark:
if blocks_written >= fsync_last + self._dest_fsync_watermark:
fsync_last = blocks_written
self.sync()
try:
self._f_dest.write(buf)
except IOError as err:
raise Error("error while writing blocks %d-%d of '%s': %s" %
(start, end, self._dest_path, err))
self._batch_queue.task_done()
blocks_written += (end - start + 1)
bytes_written += len(buf)
self._update_progress(blocks_written)
if not self.image_size:
# The image size was unknown up until now, set it
self._set_image_size(bytes_written)
# This is just a sanity check - we should have written exactly
# 'mapped_cnt' blocks.
if blocks_written != self.mapped_cnt:
raise Error("wrote %u blocks from image '%s' to '%s', but should "
"have %u - bmap file '%s' does not belong to this "
"image" %
(blocks_written, self._image_path, self._dest_path,
self.mapped_cnt, self._bmap_path))
if self._dest_is_regfile:
# Make sure the destination file has the same size as the image
try:
os.ftruncate(self._f_dest.fileno(), self.image_size)
except OSError as err:
raise Error("cannot truncate file '%s': %s" %
(self._dest_path, err))
try:
self._f_dest.flush()
except IOError as err:
raise Error("cannot flush '%s': %s" % (self._dest_path, err))
if sync:
self.sync()
def write(self, pid):
os.ftruncate(self.fd, 0)
os.write(self.fd, b"%d" % pid)
os.fsync(self.fd)
def mksparse(outfile, size):
'''use os.ftruncate to create a sparse file of the given size.'''
fobj = open(outfile, "w")
os.ftruncate(fobj.fileno(), size)
def write(self, target, expand):
"""Write disk image to the media or file."""
def write_sfdisk_script(outf, parts):
for key, val in parts['partitiontable'].items():
if key in ("partitions", "device", "firstlba", "lastlba"):
continue
if key == "id":
key = "label-id"
outf.write("{}: {}\n".format(key, val))
outf.write("\n")
for part in parts['partitiontable']['partitions']:
line = ''
for name in ('attrs', 'name', 'size', 'type', 'uuid'):
if name == 'size' and part['type'] == 'f':
# don't write size for extended partition
continue
val = part.get(name)
if val:
line += '{}={}, '.format(name, val)
if line:
line = line[:-2] # strip ', '
if part.get('bootable'):
line += ' ,bootable'
outf.write("{}\n".format(line))
outf.flush()
def read_ptable(path):
out = exec_cmd("{} -dJ {}".format(self.sfdisk, path))
return json.loads(out)
def write_ptable(parts, target):
with tempfile.NamedTemporaryFile(prefix="wic-sfdisk-",
mode='w') as outf:
write_sfdisk_script(outf, parts)
cmd = "{} --no-reread {} < {} ".format(self.sfdisk, target,
outf.name)
exec_cmd(cmd, as_shell=True)
if expand is None:
sparse_copy(self.imagepath, target)
else:
# copy first sectors that may contain bootloader
sparse_copy(self.imagepath,
target,
length=2048 * self._lsector_size)
# copy source partition table to the target
parts = read_ptable(self.imagepath)
write_ptable(parts, target)
# get size of unpartitioned space
free = None
for line in exec_cmd("{} -F {}".format(self.sfdisk,
target)).splitlines():
if line.startswith("Unpartitioned space ") and line.endswith(
"sectors"):
free = int(line.split()[-2])
# Align free space to a 2048 sector boundary. YOCTO #12840.
free = free - (free % 2048)
if free is None:
raise WicError("Can't get size of unpartitioned space")
# calculate expanded partitions sizes
sizes = {}
num_auto_resize = 0
for num, part in enumerate(parts['partitiontable']['partitions'],
1):
if num in expand:
if expand[
num] != 0: # don't resize partition if size is set to 0
sectors = expand[num] // self._lsector_size
free -= sectors - part['size']
part['size'] = sectors
sizes[num] = sectors
elif part['type'] != 'f':
sizes[num] = -1
num_auto_resize += 1
for num, part in enumerate(parts['partitiontable']['partitions'],
1):
if sizes.get(num) == -1:
part['size'] += free // num_auto_resize
# write resized partition table to the target
write_ptable(parts, target)
# read resized partition table
parts = read_ptable(target)
# copy partitions content
for num, part in enumerate(parts['partitiontable']['partitions'],
1):
pnum = str(num)
fstype = self.partitions[pnum].fstype
# copy unchanged partition
if part['size'] == self.partitions[
pnum].size // self._lsector_size:
logger.info("copying unchanged partition {}".format(pnum))
sparse_copy(self._get_part_image(pnum),
target,
seek=part['start'] * self._lsector_size)
continue
# resize or re-create partitions
if fstype.startswith('ext') or fstype.startswith('fat') or \
fstype.startswith('linux-swap'):
partfname = None
with tempfile.NamedTemporaryFile(
prefix="wic-part{}-".format(pnum)) as partf:
partfname = partf.name
if fstype.startswith('ext'):
logger.info("resizing ext partition {}".format(pnum))
partimg = self._get_part_image(pnum)
sparse_copy(partimg, partfname)
exec_cmd("{} -pf {}".format(self.e2fsck, partfname))
exec_cmd("{} {} {}s".format(\
self.resize2fs, partfname, part['size']))
elif fstype.startswith('fat'):
logger.info(
"copying content of the fat partition {}".format(
pnum))
with tempfile.TemporaryDirectory(
prefix='wic-fatdir-') as tmpdir:
# copy content to the temporary directory
cmd = "{} -snompi {} :: {}".format(
self.mcopy, self._get_part_image(pnum), tmpdir)
exec_cmd(cmd)
# create new msdos partition
label = part.get("name")
label_str = "-n {}".format(label) if label else ''
cmd = "{} {} -C {} {}".format(
self.mkdosfs, label_str, partfname,
part['size'])
exec_cmd(cmd)
# copy content from the temporary directory to the new partition
cmd = "{} -snompi {} {}/* ::".format(
self.mcopy, partfname, tmpdir)
exec_cmd(cmd, as_shell=True)
elif fstype.startswith('linux-swap'):
logger.info("creating swap partition {}".format(pnum))
label = part.get("name")
label_str = "-L {}".format(label) if label else ''
uuid = part.get("uuid")
uuid_str = "-U {}".format(uuid) if uuid else ''
with open(partfname, 'w') as sparse:
os.ftruncate(sparse.fileno(),
part['size'] * self._lsector_size)
exec_cmd("{} {} {} {}".format(self.mkswap, label_str,
uuid_str, partfname))
sparse_copy(partfname,
target,
seek=part['start'] * self._lsector_size)
os.unlink(partfname)
elif part['type'] != 'f':
logger.warning(
"skipping partition {}: unsupported fstype {}".format(
pnum, fstype))
def copy(self, sync=True, verify=True):
""" Copy the image to the destination file using bmap. The sync
argument defines whether the destination file has to be synchronized
upon return. The 'verify' argument defines whether the SHA1 checksum
has to be verified while copying. """
# Save file positions in order to restore them at the end
image_pos = self._f_image.tell()
dest_pos = self._f_dest.tell()
if self._f_bmap:
bmap_pos = self._f_bmap.tell()
# Create the queue for block batches and start the reader thread, which
# will read the image in batches and put the results to '_batch_queue'.
self._batch_queue = Queue.Queue(self._batch_queue_len)
thread.start_new_thread(self._get_data, (verify, ))
blocks_written = 0
bytes_written = 0
fsync_last = 0
# Read the image in '_batch_blocks' chunks and write them to the
# destination file
while True:
batch = self._batch_queue.get()
if batch is None:
# No more data, the image is written
break
elif batch[0] == "error":
# The reader thread encountered an error and passed us the
# exception.
exc_info = batch[1]
raise exc_info[1].with_traceback(exc_info[2])
(start, end, buf) = batch[1:4]
assert len(buf) <= (end - start + 1) * self.block_size
assert len(buf) > (end - start) * self.block_size
self._f_dest.seek(start * self.block_size)
# Synchronize the destination file if we reached the watermark
if self._dest_fsync_watermark:
if blocks_written >= fsync_last + self._dest_fsync_watermark:
fsync_last = blocks_written
self.sync()
try:
self._f_dest.write(buf)
except IOError as err:
raise Error("error while writing blocks %d-%d of '%s': %s" \
% (start, end, self._dest_path, err))
self._batch_queue.task_done()
blocks_written += (end - start + 1)
bytes_written += len(buf)
if not self.image_size:
# The image size was unknown up until now, probably because this is
# a compressed image. Initialize the corresponding class attributes
# now, when we know the size.
self._initialize_sizes(bytes_written)
# This is just a sanity check - we should have written exactly
# 'mapped_cnt' blocks.
if blocks_written != self.mapped_cnt:
raise Error("wrote %u blocks, but should have %u - inconsistent " \
"bmap file" % (blocks_written, self.mapped_cnt))
if self._dest_is_regfile:
# Make sure the destination file has the same size as the image
try:
os.ftruncate(self._f_dest.fileno(), self.image_size)
except OSError as err:
raise Error("cannot truncate file '%s': %s" \
% (self._dest_path, err))
try:
self._f_dest.flush()
except IOError as err:
raise Error("cannot flush '%s': %s" % (self._dest_path, err))
if sync:
self.sync()
# Restore file positions
self._f_image.seek(image_pos)
self._f_dest.seek(dest_pos)
if self._f_bmap:
self._f_bmap.seek(bmap_pos)
async def resize(self, length):
await self.file_transactions.fd_resize(self.fd, length)
os.ftruncate(self.file_oracle_fd, length)
def padToBlockSize(path):
with open(path, 'a') as f:
size = os.fstat(f.fileno()).st_size
newSize = 512 * ((size + 511) / 512)
log.debug("Truncating file %s to %d bytes", path, newSize)
os.ftruncate(f.fileno(), newSize)
def __init__(self, name=None, create=False, size=0):
if not size >= 0:
raise ValueError("'size' must be a positive integer")
if create:
self._flags = _O_CREX | os.O_RDWR
if size == 0:
raise ValueError(
"'size' must be a positive number different from zero")
if name is None and not self._flags & os.O_EXCL:
raise ValueError("'name' can only be None if create=True")
if _USE_POSIX:
# POSIX Shared Memory
if name is None:
while True:
name = _make_filename()
try:
self._fd = _posixshmem.shm_open(name,
self._flags,
mode=self._mode)
except FileExistsError:
continue
self._name = name
break
else:
name = "/" + name if self._prepend_leading_slash else name
self._fd = _posixshmem.shm_open(name,
self._flags,
mode=self._mode)
self._name = name
try:
if create and size:
os.ftruncate(self._fd, size)
stats = os.fstat(self._fd)
size = stats.st_size
self._mmap = mmap.mmap(self._fd, size)
except OSError:
self.unlink()
raise
resource_tracker.register(self._name, "shared_memory")
else:
# Windows Named Shared Memory
if create:
while True:
temp_name = _make_filename() if name is None else name
# Create and reserve shared memory block with this name
# until it can be attached to by mmap.
h_map = _winapi.CreateFileMapping(
_winapi.INVALID_HANDLE_VALUE, _winapi.NULL,
_winapi.PAGE_READWRITE, (size >> 32) & 0xFFFFFFFF,
size & 0xFFFFFFFF, temp_name)
try:
last_error_code = _winapi.GetLastError()
if last_error_code == _winapi.ERROR_ALREADY_EXISTS:
if name is not None:
raise FileExistsError(
errno.EEXIST, os.strerror(errno.EEXIST),
name, _winapi.ERROR_ALREADY_EXISTS)
else:
continue
self._mmap = mmap.mmap(-1, size, tagname=temp_name)
finally:
_winapi.CloseHandle(h_map)
self._name = temp_name
break
else:
self._name = name
# Dynamically determine the existing named shared memory
# block's size which is likely a multiple of mmap.PAGESIZE.
h_map = _winapi.OpenFileMapping(_winapi.FILE_MAP_READ, False,
name)
try:
p_buf = _winapi.MapViewOfFile(h_map, _winapi.FILE_MAP_READ,
0, 0, 0)
finally:
_winapi.CloseHandle(h_map)
size = _winapi.VirtualQuerySize(p_buf)
self._mmap = mmap.mmap(-1, size, tagname=name)
self._size = size
self._buf = memoryview(self._mmap)
#!/usr/bin/python3
import os, sys
#打开文件
fd =os.open("foo.txt",os.O_RDWR|os.O_CREAT)
#写入字符串
os.write(fd,"This is test - This is test".encode())
#使用ftruncate()方法
os.ftruncate(fd,10)
#读取内容
os.lseek(fd,0,0)
str=os.read(fd,100)
print("读取的字符串是：",str)
#关闭文件
os.close(fd)
print("关闭文件成功！！")
def replaceFdContent(fd, content):
os.lseek(fd, 0, os.SEEK_SET)
os.ftruncate(fd, 0)
os.write(fd, bytes(content, "utf-8"))
def rollback(self):
"""truncate file to last_offset."""
os.ftruncate(self.last_offset)
def CreateEmptyFile(fd, size):
if (size > 0):
os.ftruncate(fd, size)
#!/usr/bin/env python
import os
import mmap
bin = file("/usr/bin/id").read()
fd = os.open("id", os.O_RDWR | os.O_CREAT | os.O_EXCL, 02750)
os.ftruncate(fd, len(bin))
m = mmap.mmap(fd, len(bin))
m[:] = bin
m.flush()
def delete_content(fd):
os.ftruncate(fd, 0)
os.lseek(fd, 0, os.SEEK_SET)
def init_shmem_file(filepath, size):
with open(filepath, "w+b") as f:
os.ftruncate(f.fileno(), size)
def _do_file(self):
"""Operate on a file, create, read, truncate, etc."""
self.absfile = ""
# Number of files available
nlen = len(self.n_files)
if self.up_minfiles == 0 and nlen > self.minfiles:
self.minfiles = self.bot_minfiles
self.up_minfiles = 1
if self.up_minfiles > 0 and nlen < self.minfiles:
self.minfiles = self.top_minfiles
self.up_minfiles = 0
if nlen > self.minfiles and self._percent(self.trunc):
# Truncate file using the file name
fileobj = self._get_fileobj()
self.absfile = os.path.join(self.datadir, fileobj.name)
# Choose new size at random
nsize = self.random.randint(0, fileobj.size + self.wsizedev)
self._dprint(
"DBG2",
"TRUNC %s %d -> %d" % (fileobj.name, fileobj.size, nsize))
out = self.libc.truncate(self.absfile, nsize)
if out == -1:
err = ctypes.get_errno()
if hasattr(fileobj, 'srcname') and err == errno.ENOENT:
# Make sure not to fail if it is a broken symbolic link
self._dprint(
"DBG2",
"TRUNC %s: broken symbolic link" % fileobj.name)
return
raise OSError(err, os.strerror(err), fileobj.name)
else:
self.ntrunc += 1
fileobj.size = nsize
return
if nlen > self.minfiles and self._percent(self.rename):
# Rename file
fileobj = self._get_fileobj()
name = self._newname()
self.absfile = os.path.join(self.datadir, fileobj.name)
newfile = os.path.join(self.datadir, name)
self._dprint("DBG2", "RENAME %s -> %s" % (fileobj.name, name))
os.rename(self.absfile, newfile)
self.nrename += 1
fileobj.name = name
return
if nlen > self.minfiles and self._percent(self.remove):
# Remove file
fileobj = self._get_fileobj()
self.absfile = os.path.join(self.datadir, fileobj.name)
self._dprint("DBG2", "REMOVE %s" % fileobj.name)
os.unlink(self.absfile)
self.nremove += 1
self.n_files.remove(fileobj)
return
if nlen > self.minfiles and self._percent(self.link):
# Create hard link
name = self._newname()
self.absfile = os.path.join(self.datadir, name)
try:
fileobj = self._get_fileobj(
condition=lambda f: not hasattr(f, 'srcname'))
except NoFilesMatchingCondition:
raise Exception(
"Unable to find a valid source file for hard link")
srcfile = os.path.join(self.datadir, fileobj.name)
self._dprint("DBG2", "LINK %s -> %s" % (name, fileobj.name))
os.link(srcfile, self.absfile)
self.nlink += 1
linkobj = FileObj(name=name, size=fileobj.size)
self.n_files.append(linkobj)
return
if nlen > self.minfiles and self._percent(self.slink):
# Create symbolic link
name = self._newname()
self.absfile = os.path.join(self.datadir, name)
try:
fileobj = self._get_fileobj(
condition=lambda f: not hasattr(f, 'srcname'))
except NoFilesMatchingCondition:
raise Exception(
"Unable to find a valid source file for symbolic link")
self._dprint("DBG2", "SLINK %s -> %s" % (name, fileobj.name))
os.symlink(fileobj.name, self.absfile)
self.nslink += 1
slinkobj = FileObj(name=name,
size=fileobj.size,
srcname=fileobj.name)
self.n_files.append(slinkobj)
return
if nlen > self.minfiles and self._percent(self.readdir):
# Read directory
count = self.random.randint(1, 99)
self._dprint("DBG2",
"READDIR %s maxentries: %d" % (self.datadir, count))
self.absfile = self.datadir
fd = self.libc.opendir(self.datadir)
index = 0
while True:
dirent = self.libc.readdir(fd)
if dirent == 0 or index >= count:
break
index += 1
out = self.libc.closedir(fd)
self.nreaddir += 1
return
# Select type of open: read, write or rdwr
total = self.read + self.write
rn = self.random.randint(0, 99)
if rn < self.read:
oflags = os.O_RDONLY
oflist = ["O_RDONLY"]
elif rn < total:
oflags = os.O_WRONLY
oflist = ["O_WRONLY"]
else:
oflags = os.O_RDWR
oflist = ["O_RDWR"]
# Set create file flag
if nlen < self.minfiles:
# Create at least self.minfiles before any other operation
cflag = True
else:
cflag = self._percent(self.create)
if cflag:
# Create new name
name = self._newname()
fileobj = FileObj(name=name, size=0)
self.n_files.append(fileobj)
if oflags == os.O_RDONLY:
# Creating file, must be able to write
oflags = os.O_WRONLY
oflist = ["O_WRONLY"]
oflags |= os.O_CREAT
oflist.append("O_CREAT")
else:
# Use name chosen at random
fileobj = self._get_fileobj()
if "O_RDONLY" not in oflist and self._percent(self.osync):
# Add O_SYNC flag when opening file for writing
oflags |= os.O_SYNC
oflist.append("O_SYNC")
self.nosync += 1
if self.direct:
# Open file for direct I/O
oflags |= os.O_DIRECT
oflist.append("O_DIRECT")
# Select random or sequential I/O
sstr = "sequen"
if self._percent(self.randio):
sstr = "random"
ostr = "|".join(oflist)
fd = None
index = 0
is_symlink = False
while fd is None:
try:
index += 1
if hasattr(fileobj, 'srcname'):
is_symlink = True
self.absfile = os.path.join(self.datadir, fileobj.name)
self._dprint("DBG2",
"OPEN %s %s %s" % (fileobj.name, sstr, ostr))
fd = os.open(self.absfile, oflags)
st = os.fstat(fd)
if is_symlink:
self._dprint(
"DBG6", "OPEN %s inode:%d symlink" %
(fileobj.name, st.st_ino))
absfile = os.path.join(self.datadir, fileobj.srcname)
st = os.stat(absfile)
self._dprint(
"DBG6", "OPEN %s inode:%d src:%s" %
(fileobj.name, st.st_ino, fileobj.srcname))
else:
self._dprint(
"DBG6",
"OPEN %s inode:%d" % (fileobj.name, st.st_ino))
except OSError as openerr:
if is_symlink and openerr.errno == errno.ENOENT:
self._dprint(
"DBG2",
"OPEN %s: broken symbolic link" % fileobj.name)
if index >= 10:
# Do not exit execution, just return to select another operation
return
# Choose a new name at random
fileobj = self._get_fileobj()
is_symlink = False
else:
# Unknown error
raise
self.nopen += 1
# Get file size for writing
size = int(abs(self.random.gauss(self.fsizeavg, self.fsizedev)))
odgrade = False
if oflags & os.O_WRONLY == os.O_WRONLY:
lock_type = fcntl.F_WRLCK
iolist = self._getiolist(size, True)
elif oflags & os.O_RDWR == os.O_RDWR:
lock_type = None
iolist = self._getiolist(size, True)
iolist += self._getiolist(size, False)
if self._percent(self.odgrade):
odgrade = True
else:
lock_type = fcntl.F_RDLCK
size = fileobj.size
if size == 0:
# File does not have any data, at least try to read one block
size = self.rsize
iolist = self._getiolist(size, False)
if sstr == "random":
# Shuffle I/O list for random access
self.random.shuffle(iolist)
# Lock full file if necessary
lockfull = False
if self._percent(self.lockfull):
lockfull = True
lockfout = self._getlock(fileobj.name,
fd,
lock_type=lock_type,
offset=0,
length=0)
if nlen > self.minfiles and "O_RDONLY" not in oflist and self._percent(
self.ftrunc):
# Truncate file using the file descriptor
# Choose new size at random
nsize = self.random.randint(0, fileobj.size + self.wsizedev)
self._dprint(
"DBG2",
"FTRUNC %s %d -> %d" % (fileobj.name, fileobj.size, nsize))
os.ftruncate(fd, nsize)
self.nftrunc += 1
fileobj.size = nsize
# Read or write the file
for item in iolist:
if self.runtime > 0 and time.time() >= self.s_time + self.runtime:
# Runtime has been reached
break
self._do_io(
**dict(fd=fd, fileobj=fileobj, lockfull=lockfull, **item))
if lockfull:
# Unlock full file
self._getlock(fileobj.name,
fd,
lock_type=fcntl.F_UNLCK,
offset=0,
length=0,
lock=lockfout)
fdr = None
fdroffset = 0
if odgrade:
# Need for open downgrade:
# First, the file has been opened for read and write
# Second, open file again for reading
# Then close read and write file descriptor
self._dprint("DBG2", "OPENDGR %s" % fileobj.name)
fdr = os.open(self.absfile, os.O_RDONLY)
self.nopendgr += 1
count = self._do_io(fd=fdr,
offset=fdroffset,
size=self.rsize,
fileobj=fileobj)
fdroffset += count
# Close main file descriptor
self._dprint("DBG3", "CLOSE %s" % fileobj.name)
os.close(fd)
self.nclose += 1
if odgrade:
for i in xrange(10):
count = self._do_io(fd=fdr,
offset=fdroffset,
size=self.rsize,
fileobj=fileobj)
fdroffset += count
self._dprint("DBG3", "CLOSE %s" % fileobj.name)
os.close(fdr)
self.nclose += 1
return
def recvSkipHandler(stream, length, opaque):
fd = opaque
cur = os.lseek(fd, length, os.SEEK_CUR)
return os.ftruncate(fd, cur)
def _clone_local(self, meter, size_bytes):
if self._input_path == "/dev/null": # pragma: no cover
# Not really sure why this check is here,
# but keeping for compat
log.debug("Source dev was /dev/null. Skipping")
return
if self._input_path == self._output_path:
log.debug("Source and destination are the same. Skipping.")
return
# If a destination file exists and sparse flag is True,
# this priority takes an existing file.
if (not os.path.exists(self._output_path) and self._sparse):
clone_block_size = 4096
sparse = True
fd = None
try:
fd = os.open(self._output_path, os.O_WRONLY | os.O_CREAT,
0o640)
os.ftruncate(fd, size_bytes)
finally:
if fd:
os.close(fd)
else:
clone_block_size = 1024 * 1024 * 10
sparse = False
log.debug("Local Cloning %s to %s, sparse=%s, block_size=%s",
self._input_path, self._output_path,
sparse, clone_block_size)
zeros = b'\0' * 4096
src_fd, dst_fd = None, None
try:
try:
src_fd = os.open(self._input_path, os.O_RDONLY)
dst_fd = os.open(self._output_path,
os.O_WRONLY | os.O_CREAT, 0o640)
if (True):
#size = int(self.get_size() * 1024 * 1024 * 1024 * 10)
size = _get_size(self._path)
log.debug(("USING REF {size}", size), exc_info=True)
while size > 0:
ret = os.copy_file_range(src_fd, dst_fd, size)
size -= ret
else:
i = 0
while 1:
l = os.read(src_fd, clone_block_size)
s = len(l)
if s == 0:
meter.end(size_bytes)
break
# check sequence of zeros
if sparse and zeros == l:
os.lseek(dst_fd, s, 1)
else:
b = os.write(dst_fd, l)
if s != b: # pragma: no cover
meter.end(i)
break
i += s
if i < size_bytes:
meter.update(i)
except OSError as e: # pragma: no cover
log.debug("Error while cloning", exc_info=True)
msg = (_("Error cloning diskimage "
"%(inputpath)s to %(outputpath)s: %(error)s") %
{"inputpath": self._input_path,
"outputpath": self._output_path,
"error": str(e)})
raise RuntimeError(msg) from None
finally:
if src_fd is not None:
os.close(src_fd)
if dst_fd is not None:
os.close(dst_fd)
def _truncate(self, size):
os.ftruncate(self.fd, size)
def __init__(self, seed, enable_graphviz=False, flush=True):
global KaflNodeID
self.level = 0
self.max_level = 0
self.cycles = -1
self.all_nodes = []
self.references = {}
self.current = self.MASTER_NODE_ID
self.random_shuffled = False
self.favorite_buf = []
self.favorite_unfinished_buf = []
self.regular_buf = []
self.regular_unfinished_buf = []
self.finished_buf = []
self.bitmap_size = FuzzerConfiguration(
).config_values['BITMAP_SHM_SIZE']
self.buckets = [0x0, 0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80]
self.fav_bitmap = []
self.fav_bitmap_updated = False
self.bitmap_fd = os.open(
FuzzerConfiguration().argument_values['work_dir'] + "/bitmap",
os.O_RDWR | os.O_SYNC | os.O_CREAT)
self.crash_bitmap_fd = os.open(
FuzzerConfiguration().argument_values['work_dir'] +
"/crash_bitmap", os.O_RDWR | os.O_SYNC | os.O_CREAT)
self.kasan_bitmap_fd = os.open(
FuzzerConfiguration().argument_values['work_dir'] +
"/kasan_bitmap", os.O_RDWR | os.O_SYNC | os.O_CREAT)
self.timeout_bitmap_fd = os.open(
FuzzerConfiguration().argument_values['work_dir'] +
"/timeout_bitmap", os.O_RDWR | os.O_SYNC | os.O_CREAT)
os.ftruncate(self.bitmap_fd,
FuzzerConfiguration().config_values['BITMAP_SHM_SIZE'])
os.ftruncate(self.crash_bitmap_fd,
FuzzerConfiguration().config_values['BITMAP_SHM_SIZE'])
os.ftruncate(self.kasan_bitmap_fd,
FuzzerConfiguration().config_values['BITMAP_SHM_SIZE'])
os.ftruncate(self.timeout_bitmap_fd,
FuzzerConfiguration().config_values['BITMAP_SHM_SIZE'])
self.bitmap = mmap.mmap(self.bitmap_fd, self.bitmap_size,
mmap.MAP_SHARED,
mmap.PROT_WRITE | mmap.PROT_READ)
self.crash_bitmap = mmap.mmap(self.crash_bitmap_fd, self.bitmap_size,
mmap.MAP_SHARED,
mmap.PROT_WRITE | mmap.PROT_READ)
self.kasan_bitmap = mmap.mmap(self.kasan_bitmap_fd, self.bitmap_size,
mmap.MAP_SHARED,
mmap.PROT_WRITE | mmap.PROT_READ)
self.timeout_bitmap = mmap.mmap(self.timeout_bitmap_fd,
self.bitmap_size, mmap.MAP_SHARED,
mmap.PROT_WRITE | mmap.PROT_READ)
if flush:
for i in range(self.bitmap_size):
self.bitmap[i] = '\x00'
self.crash_bitmap[i] = '\x00'
self.kasan_bitmap[i] = '\x00'
self.timeout_bitmap[i] = '\x00'
for i in range(self.bitmap_size):
self.fav_bitmap.append(None)
self.graph = KaflGraph([], enabled=enable_graphviz)
self.favorites = 0
self.favorites_in_progress = 0
self.favorites_finished = 0
self.paths = 0
self.paths = len(seed)
self.paths_in_progress = 0
self.paths_finished = 0
self.score_changed = False
self.payload_hashes = {}
for payload, bitmap in seed:
node = KaflNode(self.level,
payload,
bitmap,
node_type=KaflNodeType.favorite)
self.__append_to_level(node)
self.favorites += 1
if self.all_nodes:
self.current = 0 #self.__get_ref(self.all_nodes[0])
self.__restore_state()
self.__restore_graph()
def create(self):
logger.debug("Creating sparse file %s", self.path)
with open(self.path, 'w') as sparse:
os.ftruncate(sparse.fileno(), self.min_size)
logger.debug("Initializing partition table for %s", self.path)
exec_native_cmd("parted -s %s mklabel %s" %
(self.path, self.ptable_format), self.native_sysroot)
logger.debug("Set disk identifier %x", self.identifier)
with open(self.path, 'r+b') as img:
img.seek(0x1B8)
img.write(self.identifier.to_bytes(4, 'little'))
logger.debug("Creating partitions")
for part in self.partitions:
if part.num == 0:
continue
if self.ptable_format == "msdos" and part.num == 5:
# Create an extended partition (note: extended
# partition is described in MBR and contains all
# logical partitions). The logical partitions save a
# sector for an EBR just before the start of a
# partition. The extended partition must start one
# sector before the start of the first logical
# partition. This way the first EBR is inside of the
# extended partition. Since the extended partitions
# starts a sector before the first logical partition,
# add a sector at the back, so that there is enough
# room for all logical partitions.
self._create_partition(self.path, "extended",
None, part.start - 1,
self.offset - part.start + 1)
if part.fstype == "swap":
parted_fs_type = "linux-swap"
elif part.fstype == "vfat":
parted_fs_type = "fat32"
elif part.fstype == "msdos":
parted_fs_type = "fat16"
if not part.system_id:
part.system_id = '0x6' # FAT16
else:
# Type for ext2/ext3/ext4/btrfs
parted_fs_type = "ext2"
# Boot ROM of OMAP boards require vfat boot partition to have an
# even number of sectors.
if part.mountpoint == "/boot" and part.fstype in ["vfat", "msdos"] \
and part.size_sec % 2:
logger.debug("Subtracting one sector from '%s' partition to "
"get even number of sectors for the partition",
part.mountpoint)
part.size_sec -= 1
self._create_partition(self.path, part.type,
parted_fs_type, part.start, part.size_sec)
if part.part_name:
logger.debug("partition %d: set name to %s",
part.num, part.part_name)
exec_native_cmd("sgdisk --change-name=%d:%s %s" % \
(part.num, part.part_name,
self.path), self.native_sysroot)
if part.part_type:
logger.debug("partition %d: set type UID to %s",
part.num, part.part_type)
exec_native_cmd("sgdisk --typecode=%d:%s %s" % \
(part.num, part.part_type,
self.path), self.native_sysroot)
if part.uuid and self.ptable_format == "gpt":
logger.debug("partition %d: set UUID to %s",
part.num, part.uuid)
exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \
(part.num, part.uuid, self.path),
self.native_sysroot)
if part.label and self.ptable_format == "gpt":
logger.debug("partition %d: set name to %s",
part.num, part.label)
exec_native_cmd("parted -s %s name %d %s" % \
(self.path, part.num, part.label),
self.native_sysroot)
if part.active:
flag_name = "legacy_boot" if self.ptable_format == 'gpt' else "boot"
logger.debug("Set '%s' flag for partition '%s' on disk '%s'",
flag_name, part.num, self.path)
exec_native_cmd("parted -s %s set %d %s on" % \
(self.path, part.num, flag_name),
self.native_sysroot)
if part.system_id:
exec_native_cmd("sfdisk --part-type %s %s %s" % \
(self.path, part.num, part.system_id),
self.native_sysroot)
def ar_open(root, path, flags, mode=None, create=False, truncate=False):
"""A function similar to os.open() that ensures that the path
we're accessing resides within a specified directory subtree.
'root' is a directory that path must reside in.
'path' is a path that is interpreted relative to 'root'. i.e., 'root'
is prepended to path. 'path' can not contain any symbolic links that
would cause an access to be redirected outside of 'root'. If this
happens we'll raise an OSError exception with errno set to EREMOTE
'mode' optional permissions mask used if we create 'path'
'create' optional flag indicating if we should create 'path'
'truncate' optional flag indicating if we should truncate 'path' after
opening it."""
# all paths must be absolute
assert os.path.isabs(root)
# only allow read/write flags
assert (flags & ~(os.O_WRONLY | os.O_RDONLY)) == 0
# we can't truncate a file unless we open it for writing
assert not truncate or (flags & os.O_WRONLY)
# if create is true the user must supply a mode mask
assert not create or mode != None
# we're going to update root and path so prepare an error
# message with the existing values now.
eremote = _("Path outside alternate root: root={root}, "
"path={path}").format(root=root, path=path)
# make target into a relative path
if os.path.isabs(path):
path = __path_abs_to_relative(path)
# now open the alternate root and get its path
# done to eliminate any links/mounts/etc in the path
root_fd = os.open(root, os.O_RDONLY)
try:
root = __fd_to_path(root_fd)
except OSError as e:
if e.errno != errno.ENOENT:
os.close(root_fd)
raise e
os.close(root_fd)
# now open the target file, get its path, and make sure it
# lives in the alternate root
path_fd = None
try:
path_tmp = os.path.join(root, path)
path_fd = os.open(path_tmp, flags)
except OSError as e:
if e.errno != errno.ENOENT or not create:
raise e
assert path_fd or create
if not path_fd:
# the file doesn't exist so we should try to create it.
# we'll do this by first opening the directory which
# will contain the file and then using openat within
# that directory.
path_dir = os.path.dirname(path)
path_file = os.path.basename(path)
try:
path_dir_fd = \
ar_open(root, path_dir, os.O_RDONLY)
except OSError as e:
if e.errno != errno.EREMOTE:
raise e
raise OSError(errno.EREMOTE, eremote)
# we opened the directory, now create the file
try:
path_fd = sat.openat(path_dir_fd, path_file,
flags | os.O_CREAT | os.O_EXCL, mode)
except OSError as e:
os.close(path_dir_fd)
raise e
# we created the file
assert path_fd
os.close(path_dir_fd)
# verify that the file we opened lives in the alternate root
try:
path = __fd_to_path(path_fd)
except OSError as e:
if e.errno != errno.ENOENT:
os.close(path_fd)
raise e
path = os.path.join(root, path)
if not path.startswith(root):
os.close(path_fd)
raise OSError(errno.EREMOTE, eremote)
if truncate:
# the user wanted us to truncate the file
try:
os.ftruncate(path_fd, 0)
except OSError as e:
os.close(path_fd)
raise e
return path_fd
process_running = False
try:
process.wait(timeout=2.0 if prev_line_count == 0 else 15.0)
except subprocess.TimeoutExpired:
process_running = True
with open(local_filename, "rt") as f:
# read new lines
new_lines = f.readlines()
if not new_lines:
continue
output = "".join(new_lines)
print(output)
# update task comment with jupyter notebook server links
if prev_line_count == 0:
task.comment += "\n" + "".join(
line for line in new_lines
if "http://" in line or "https://" in line)
prev_line_count += len(new_lines)
os.lseek(fd, 0, 0)
os.ftruncate(fd, 0)
# cleanup
os.close(fd)
# noinspection PyBroadException
try:
os.unlink(local_filename)
except Exception:
pass
def write(self, data):
os.lseek(self.fd, 0, os.SEEK_SET)
os.ftruncate(self.fd, 0)
os.write(self.fd, data)
os.fsync(self.fd)
def _truncate(fn, length):
fd = os.open(fn, os.O_RDWR)
os.ftruncate(fd, length)
os.close(fd)
'''
概述
os.ftruncate() 裁剪文件描述符fd对应的文件, 它最大不能超过文件大小。
Unix上可用。
语法
ftruncate()方法语法格式如下：
os.ftruncate(fd, length)¶
参数
fd -- 文件的描述符。
length -- 要裁剪文件大小。
返回值
该方法没有返回值。
'''
# 打开文件
fd = os.open('/tmp/foo.txt', os.O_RDWR)
# 写入内容
os.write(fd, '新生代文本内容')
# 截取文件内容
os.ftruncate(fd, 20)
# 移动文件指针
os.lseek(fd, 0, 0)
print('读取文件内容：%s' % os.read(fd, 100))
os.close(fd)
def decode_xmlrpc_chunk(chunk_start,
chunk_len,
chunk_checksum,
encoded_chunk,
write_to=None,
mode=0o644):
"""
Decode a data chunk and optionally write it to a file.
@param chunk_start: chunk start position in the file (-1 for append)
@type chunk_start: str
@param chunk_len: chunk length
@type chunk_len: str
@param chunk_checksum: sha256 checksum (lower case)
@type chunk_checksum: str
@param encoded_chunk: base64 encoded chunk
@type encoded_chunk: str
@param write_to: path to a file in which the decoded data will be written
@type write_to: str
@param mode: file permissions (example: 0644)
@type mode: int
@return: decoded data
@rtype: str
"""
chunk_start = int(chunk_start)
chunk_len = int(chunk_len)
decode_func = base64.decodebytes if hasattr(
base64, "decodebytes") else base64.decodestring
if isinstance(encoded_chunk, xmlrpclib.Binary):
chunk = decode_func(encoded_chunk.data)
else:
chunk = decode_func(encoded_chunk)
if chunk_len not in (-1, len(chunk)):
raise ValueError("Chunk length doesn't match.")
if chunk_len == -1:
chunk = ""
elif chunk_checksum != hashlib.sha256(chunk).hexdigest().lower():
raise ValueError("Chunk checksum doesn't match.")
if not write_to:
return chunk
# code below handles writing to a file
target_dir = os.path.dirname(write_to)
if not os.path.isdir(target_dir):
try:
os.makedirs(target_dir, mode=0o755)
except OSError as ex:
if ex.errno != 17:
raise
fd = os.open(write_to, os.O_RDWR | os.O_CREAT, mode)
fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
try:
if chunk_start != -1:
os.ftruncate(fd, chunk_start)
os.lseek(fd, 0, 2) # 2=os.SEEK_END
os.write(fd, chunk)
finally:
fcntl.lockf(fd, fcntl.LOCK_UN)
os.close(fd)
if chunk_start != -1 and chunk_len == -1:
# final chunk, compute checksum of whole file
file_checksum = kobo.shortcuts.compute_file_checksums(
write_to, ["sha256"])["sha256"]
if file_checksum != chunk_checksum:
raise ValueError("File checksum does not match.")
return chunk
def write_data(self, data, thefile, shred=True, mode=0o600):
# TODO: add docstrings for arg types since this code is picky about that
"""Write the data bytes to given path
This is used to write a byte string to a file or stdout. It is used for
writing the results of vault encryption or decryption. It is used for
saving the ciphertext after encryption and it is also used for saving the
plaintext after decrypting a vault. The type of the 'data' arg should be bytes,
since in the plaintext case, the original contents can be of any text encoding
or arbitrary binary data.
When used to write the result of vault encryption, the val of the 'data' arg
should be a utf-8 encoded byte string and not a text typ and not a text type..
When used to write the result of vault decryption, the val of the 'data' arg
should be a byte string and not a text type.
:arg data: the byte string (bytes) data
:arg thefile: file descriptor or filename to save 'data' to.
:arg shred: if shred==True, make sure that the original data is first shredded so that is cannot be recovered.
:returns: None
"""
# FIXME: do we need this now? data_bytes should always be a utf-8 byte string
b_file_data = to_bytes(data, errors='strict')
# check if we have a file descriptor instead of a path
is_fd = False
try:
is_fd = (isinstance(thefile, int) and fcntl.fcntl(thefile, fcntl.F_GETFD) != -1)
except Exception:
pass
if is_fd:
# if passed descriptor, use that to ensure secure access, otherwise it is a string.
# assumes the fd is securely opened by caller (mkstemp)
os.ftruncate(thefile, 0)
os.write(thefile, b_file_data)
elif thefile == '-':
# get a ref to either sys.stdout.buffer for py3 or plain old sys.stdout for py2
# We need sys.stdout.buffer on py3 so we can write bytes to it since the plaintext
# of the vaulted object could be anything/binary/etc
output = getattr(sys.stdout, 'buffer', sys.stdout)
output.write(b_file_data)
else:
# file names are insecure and prone to race conditions, so remove and create securely
if os.path.isfile(thefile):
if shred:
self._shred_file(thefile)
else:
os.remove(thefile)
# when setting new umask, we get previous as return
current_umask = os.umask(0o077)
try:
try:
# create file with secure permissions
fd = os.open(thefile, os.O_CREAT | os.O_EXCL | os.O_RDWR | os.O_TRUNC, mode)
except OSError as ose:
# Want to catch FileExistsError, which doesn't exist in Python 2, so catch OSError
# and compare the error number to get equivalent behavior in Python 2/3
if ose.errno == errno.EEXIST:
raise AnsibleError('Vault file got recreated while we were operating on it: %s' % to_native(ose))
raise AnsibleError('Problem creating temporary vault file: %s' % to_native(ose))
try:
# now write to the file and ensure ours is only data in it
os.ftruncate(fd, 0)
os.write(fd, b_file_data)
except OSError as e:
raise AnsibleError('Unable to write to temporary vault file: %s' % to_native(e))
finally:
# Make sure the file descriptor is always closed and reset umask
os.close(fd)
finally:
os.umask(current_umask)
