def _parseAttributes(self, data):
flags ,= struct.unpack('!L', data[:4])
attrs = {}
data = data[4:]
if flags & FILEXFER_ATTR_SIZE == FILEXFER_ATTR_SIZE:
size ,= struct.unpack('!Q', data[:8])
attrs['size'] = size
data = data[8:]
if flags & FILEXFER_ATTR_OWNERGROUP == FILEXFER_ATTR_OWNERGROUP:
uid, gid = struct.unpack('!2L', data[:8])
attrs['uid'] = uid
attrs['gid'] = gid
data = data[8:]
if flags & FILEXFER_ATTR_PERMISSIONS == FILEXFER_ATTR_PERMISSIONS:
perms ,= struct.unpack('!L', data[:4])
attrs['permissions'] = perms
data = data[4:]
if flags & FILEXFER_ATTR_ACMODTIME == FILEXFER_ATTR_ACMODTIME:
atime, mtime = struct.unpack('!2L', data[:8])
attrs['atime'] = atime
attrs['mtime'] = mtime
data = data[8:]
if flags & FILEXFER_ATTR_EXTENDED == FILEXFER_ATTR_EXTENDED:
extended_count ,= struct.unpack('!L', data[:4])
data = data[4:]
for i in xrange(extended_count):
extended_type, data = getNS(data)
extended_data, data = getNS(data)
attrs['ext_%s' % nativeString(extended_type)] = extended_data
return attrs, data
def render():
child = Elem()
for _ in xrange(20):
child = Elem([child])
root = TagLoader([child] * 10).load()
out = BytesIO()
flatten(None, root, out.write)
def _unzipIterChunkyTest(self, compression, chunksize, lower, upper):
"""
unzipIterChunky should unzip the given number of bytes per iteration.
"""
junk = ' '.join([str(random.random()) for n in xrange(1000)])
junkmd5 = md5(junk).hexdigest()
tempdir = filepath.FilePath(self.mktemp())
tempdir.makedirs()
zfpath = tempdir.child('bigfile.zip').path
self._makebigfile(zfpath, compression, junk)
uziter = zipstream.unzipIterChunky(zfpath,
tempdir.path,
chunksize=chunksize)
r = uziter.next()
# test that the number of chunks is in the right ballpark;
# this could theoretically be any number but statistically it
# should always be in this range
approx = lower < r < upper
self.assertTrue(approx)
for r in uziter:
pass
self.assertEqual(r, 0)
newmd5 = md5(tempdir.child("zipstreamjunk").open().read()).hexdigest()
self.assertEqual(newmd5, junkmd5)
def _parseAttributes(self, data):
flags, = struct.unpack('!L', data[:4])
attrs = {}
data = data[4:]
if flags & FILEXFER_ATTR_SIZE == FILEXFER_ATTR_SIZE:
size, = struct.unpack('!Q', data[:8])
attrs['size'] = size
data = data[8:]
if flags & FILEXFER_ATTR_OWNERGROUP == FILEXFER_ATTR_OWNERGROUP:
uid, gid = struct.unpack('!2L', data[:8])
attrs['uid'] = uid
attrs['gid'] = gid
data = data[8:]
if flags & FILEXFER_ATTR_PERMISSIONS == FILEXFER_ATTR_PERMISSIONS:
perms, = struct.unpack('!L', data[:4])
attrs['permissions'] = perms
data = data[4:]
if flags & FILEXFER_ATTR_ACMODTIME == FILEXFER_ATTR_ACMODTIME:
atime, mtime = struct.unpack('!2L', data[:8])
attrs['atime'] = atime
attrs['mtime'] = mtime
data = data[8:]
if flags & FILEXFER_ATTR_EXTENDED == FILEXFER_ATTR_EXTENDED:
extended_count, = struct.unpack('!L', data[:4])
data = data[4:]
for i in xrange(extended_count):
extended_type, data = getNS(data)
extended_data, data = getNS(data)
attrs['ext_%s' % nativeString(extended_type)] = extended_data
return attrs, data
def render():
child = Elem()
for _ in xrange(20):
child = Elem([child])
root = TagLoader([child] * 10).load()
out = BytesIO()
flatten(None, root, out.write)
def __init__(self, l, containerType):
"""
@param l: The list of object which may contain some not yet referenced
objects.
@param containerType: A type of container objects (e.g., C{tuple} or
C{set}).
"""
NotKnown.__init__(self)
self.containerType = containerType
self.l = l
self.locs = list(xrange(len(l)))
for idx in xrange(len(l)):
if not isinstance(l[idx], NotKnown):
self.locs.remove(idx)
else:
l[idx].addDependant(self, idx)
if not self.locs:
self.resolveDependants(self.containerType(self.l))
def _resetSignalDisposition(self):
# The Python interpreter ignores some signals, and our child
# process will inherit that behaviour. To have a child process
# that responds to signals normally, we need to reset our
# child process's signal handling (just) after we fork and
# before we execvpe.
for signalnum in xrange(1, signal.NSIG):
if signal.getsignal(signalnum) == signal.SIG_IGN:
# Reset signal handling to the default
signal.signal(signalnum, signal.SIG_DFL)
def __init__(self, l, containerType):
"""
@param l: The list of object which may contain some not yet referenced
objects.
@param containerType: A type of container objects (e.g., C{tuple} or
C{set}).
"""
NotKnown.__init__(self)
self.containerType = containerType
self.l = l
self.locs = list(xrange(len(l)))
for idx in xrange(len(l)):
if not isinstance(l[idx], NotKnown):
self.locs.remove(idx)
else:
l[idx].addDependant(self, idx)
if not self.locs:
self.resolveDependants(self.containerType(self.l))
def _resetSignalDisposition(self):
# The Python interpreter ignores some signals, and our child
# process will inherit that behaviour. To have a child process
# that responds to signals normally, we need to reset our
# child process's signal handling (just) after we fork and
# before we execvpe.
for signalnum in xrange(1, signal.NSIG):
if signal.getsignal(signalnum) == signal.SIG_IGN:
# Reset signal handling to the default
signal.signal(signalnum, signal.SIG_DFL)
def ranges_to_sequence(ranges):
"""Iterate over individual items represented in a ranges list."""
for item in ranges:
if isinstance(item, tuple):
start, end = item
if start > end:
raise ValueError("Range error %d > %d", start, end)
for item in xrange(start, end + 1):
yield item
else:
yield item
def resumeProducing(self):
"""Called when the consumer attached to me runs out of buffer.
"""
# Go backwards over the list so we can remove indexes from it as we go
for pageridx in xrange(len(self.pageProducers) - 1, -1, -1):
pager = self.pageProducers[pageridx]
pager.sendNextPage()
if not pager.stillPaging():
del self.pageProducers[pageridx]
if not self.pageProducers:
self.transport.unregisterProducer()
def test_processCommandLineArguments(self):
"""
Arguments given to spawnProcess are passed to the child process as
originally intended.
"""
us = FilePath(__file__).sibling(b"process_cli.py")
args = [
b'hello', b'"', b' \t|<>^&', br'"\\"hello\\"', br'"foo\ bar baz\""'
]
# Ensure that all non-NUL characters can be passed too.
if _PY3:
args.append("".join(map(chr, xrange(1, 255))).encode("utf8"))
else:
args.append("".join(map(chr, xrange(1, 255))))
reactor = self.buildReactor()
def processFinished(finishedArgs):
output, err, code = finishedArgs
output = output.split(b'\0')
# Drop the trailing \0.
output.pop()
self.assertEqual(args, output)
def shutdown(result):
reactor.stop()
return result
def spawnChild():
d = succeed(None)
d.addCallback(lambda dummy: utils.getProcessOutputAndValue(
pyExe, [us.path] + args, reactor=reactor))
d.addCallback(processFinished)
d.addBoth(shutdown)
reactor.callWhenRunning(spawnChild)
self.runReactor(reactor)
def test_processCommandLineArguments(self):
"""
Arguments given to spawnProcess are passed to the child process as
originally intended.
"""
us = b"twisted.internet.test.process_cli"
args = [b'hello', b'"', b' \t|<>^&', br'"\\"hello\\"', br'"foo\ bar baz\""']
# Ensure that all non-NUL characters can be passed too.
if _PY3:
args.append("".join(map(chr, xrange(1,255))).encode("utf8"))
else:
args.append("".join(map(chr, xrange(1,255))))
reactor = self.buildReactor()
def processFinished(finishedArgs):
output, err, code = finishedArgs
output = output.split(b'\0')
# Drop the trailing \0.
output.pop()
self.assertEqual(args, output)
def shutdown(result):
reactor.stop()
return result
def spawnChild():
d = succeed(None)
d.addCallback(lambda dummy: utils.getProcessOutputAndValue(
pyExe, [b"-m", us] + args, env=properEnv,
reactor=reactor))
d.addCallback(processFinished)
d.addBoth(shutdown)
reactor.callWhenRunning(spawnChild)
self.runReactor(reactor)
def send_method_call(self, method, args=[], kwargs={}):
"""Send a L{MethodCall} command with the given arguments.
If a response from the server is not received within C{self.timeout}
seconds, the returned deferred will errback with a L{MethodCallError}.
@param method: The name of the remote method to invoke.
@param args: The positional arguments to pass to the remote method.
@param kwargs: The keyword arguments to pass to the remote method.
@return: A C{Deferred} firing with the return value of the method
invoked on the remote object. If the remote method itself returns
a deferred, we fire with the callback value of such deferred.
"""
arguments = bpickle.dumps((args, kwargs))
sequence = uuid4().int
# As we send the method name to remote, we need bytes.
method = method.encode("utf-8")
# Split the given arguments in one or more chunks
chunks = [arguments[i:i + self._chunk_size]
for i in xrange(0, len(arguments), self._chunk_size)]
result = Deferred()
if len(chunks) > 1:
# If we have N chunks, send the first N-1 as MethodCallChunk's
for chunk in chunks[:-1]:
def create_send_chunk(sequence, chunk):
send_chunk = (lambda x: self._protocol.callRemote(
MethodCallChunk, sequence=sequence, chunk=chunk))
return send_chunk
result.addCallback(create_send_chunk(sequence, chunk))
def send_last_chunk(ignored):
chunk = chunks[-1]
return self._call_remote_with_timeout(
MethodCall, sequence=sequence, method=method, arguments=chunk)
result.addCallback(send_last_chunk)
result.addCallback(lambda response: response["result"])
result.callback(None)
return result
def eraseDisplay(self):
self.lines = [self._emptyLine(self.width) for i in xrange(self.height)]
def _fork(self, path, uid, gid, executable, args, environment, **kwargs):
"""
Fork and then exec sub-process.
@param path: the path where to run the new process.
@type path: C{str}
@param uid: if defined, the uid used to run the new process.
@type uid: C{int}
@param gid: if defined, the gid used to run the new process.
@type gid: C{int}
@param executable: the executable to run in a new process.
@type executable: C{str}
@param args: arguments used to create the new process.
@type args: C{list}.
@param environment: environment used for the new process.
@type environment: C{dict}.
@param kwargs: keyword arguments to L{_setupChild} method.
"""
collectorEnabled = gc.isenabled()
gc.disable()
try:
self.pid = os.fork()
except:
# Still in the parent process
if collectorEnabled:
gc.enable()
raise
else:
if self.pid == 0:
# A return value of 0 from fork() indicates that we are now
# executing in the child process.
# Do not put *ANY* code outside the try block. The child
# process must either exec or _exit. If it gets outside this
# block (due to an exception that is not handled here, but
# which might be handled higher up), there will be two copies
# of the parent running in parallel, doing all kinds of damage.
# After each change to this code, review it to make sure there
# are no exit paths.
try:
# Stop debugging. If I am, I don't care anymore.
sys.settrace(None)
self._setupChild(**kwargs)
self._execChild(path, uid, gid, executable, args,
environment)
except:
# If there are errors, try to write something descriptive
# to stderr before exiting.
# The parent's stderr isn't *necessarily* fd 2 anymore, or
# even still available; however, even libc assumes that
# write(2, err) is a useful thing to attempt.
try:
stderr = os.fdopen(2, 'wb')
msg = ("Upon execvpe {0} {1} in environment id {2}"
"\n:").format(executable, str(args),
id(environment))
if _PY3:
# On Python 3, print_exc takes a text stream, but
# on Python 2 it still takes a byte stream. So on
# Python 3 we will wrap up the byte stream returned
# by os.fdopen using TextIOWrapper.
# We hard-code UTF-8 as the encoding here, rather
# than looking at something like
# getfilesystemencoding() or sys.stderr.encoding,
# because we want an encoding that will be able to
# encode the full range of code points. We are
# (most likely) talking to the parent process on
# the other end of this pipe and not the filesystem
# or the original sys.stderr, so there's no point
# in trying to match the encoding of one of those
# objects.
stderr = io.TextIOWrapper(stderr, encoding="utf-8")
stderr.write(msg)
traceback.print_exc(file=stderr)
stderr.flush()
for fd in xrange(3):
os.close(fd)
except:
# Handle all errors during the error-reporting process
# silently to ensure that the child terminates.
pass
# See comment above about making sure that we reach this line
# of code.
os._exit(1)
# we are now in parent process
if collectorEnabled:
gc.enable()
self.status = -1 # this records the exit status of the child
def _fork(self, path, uid, gid, executable, args, environment, **kwargs):
"""
Fork and then exec sub-process.
@param path: the path where to run the new process.
@type path: C{str}
@param uid: if defined, the uid used to run the new process.
@type uid: C{int}
@param gid: if defined, the gid used to run the new process.
@type gid: C{int}
@param executable: the executable to run in a new process.
@type executable: C{str}
@param args: arguments used to create the new process.
@type args: C{list}.
@param environment: environment used for the new process.
@type environment: C{dict}.
@param kwargs: keyword arguments to L{_setupChild} method.
"""
collectorEnabled = gc.isenabled()
gc.disable()
try:
self.pid = os.fork()
except:
# Still in the parent process
if collectorEnabled:
gc.enable()
raise
else:
if self.pid == 0: # pid is 0 in the child process
# do not put *ANY* code outside the try block. The child process
# must either exec or _exit. If it gets outside this block (due
# to an exception that is not handled here, but which might be
# handled higher up), there will be two copies of the parent
# running in parallel, doing all kinds of damage.
# After each change to this code, review it to make sure there
# are no exit paths.
try:
# Stop debugging. If I am, I don't care anymore.
sys.settrace(None)
self._setupChild(**kwargs)
self._execChild(
path, uid, gid, executable, args, environment)
except:
# If there are errors, bail and try to write something
# descriptive to stderr.
# XXX: The parent's stderr isn't necessarily fd 2 anymore, or
# even still available
# XXXX: however even libc assumes write(2, err) is a useful
# thing to attempt
try:
stderr = os.fdopen(2, 'wb')
msg = ("Upon execvpe {0} {1} in environment id {2}"
"\n:").format(executable, str(args),
id(environment))
tb = NativeStringIO()
traceback.print_exc(file=tb)
tb = tb.getvalue()
if _PY3:
msg = msg.encode(sys.getfilesystemencoding())
tb = tb.encode(sys.getfilesystemencoding())
stderr.write(msg)
stderr.write(tb)
stderr.flush()
for fd in xrange(3):
os.close(fd)
except:
pass # make *sure* the child terminates
# Did you read the comment about not adding code here?
os._exit(1)
# we are now in parent process
if collectorEnabled:
gc.enable()
self.status = -1 # this records the exit status of the child
def _fork(self, path, uid, gid, executable, args, environment, **kwargs):
"""
Fork and then exec sub-process.
@param path: the path where to run the new process.
@type path: C{str}
@param uid: if defined, the uid used to run the new process.
@type uid: C{int}
@param gid: if defined, the gid used to run the new process.
@type gid: C{int}
@param executable: the executable to run in a new process.
@type executable: C{str}
@param args: arguments used to create the new process.
@type args: C{list}.
@param environment: environment used for the new process.
@type environment: C{dict}.
@param kwargs: keyword arguments to L{_setupChild} method.
"""
collectorEnabled = gc.isenabled()
gc.disable()
try:
self.pid = os.fork()
except:
# Still in the parent process
if collectorEnabled:
gc.enable()
raise
else:
if self.pid == 0:
# A return value of 0 from fork() indicates that we are now
# executing in the child process.
# Do not put *ANY* code outside the try block. The child
# process must either exec or _exit. If it gets outside this
# block (due to an exception that is not handled here, but
# which might be handled higher up), there will be two copies
# of the parent running in parallel, doing all kinds of damage.
# After each change to this code, review it to make sure there
# are no exit paths.
try:
# Stop debugging. If I am, I don't care anymore.
sys.settrace(None)
self._setupChild(**kwargs)
self._execChild(path, uid, gid, executable, args,
environment)
except:
# If there are errors, try to write something descriptive
# to stderr before exiting.
# The parent's stderr isn't *necessarily* fd 2 anymore, or
# even still available; however, even libc assumes that
# write(2, err) is a useful thing to attempt.
try:
stderr = os.fdopen(2, 'wb')
msg = ("Upon execvpe {0} {1} in environment id {2}"
"\n:").format(executable, str(args),
id(environment))
if _PY3:
# On Python 3, print_exc takes a text stream, but
# on Python 2 it still takes a byte stream. So on
# Python 3 we will wrap up the byte stream returned
# by os.fdopen using TextIOWrapper.
# We hard-code UTF-8 as the encoding here, rather
# than looking at something like
# getfilesystemencoding() or sys.stderr.encoding,
# because we want an encoding that will be able to
# encode the full range of code points. We are
# (most likely) talking to the parent process on
# the other end of this pipe and not the filesystem
# or the original sys.stderr, so there's no point
# in trying to match the encoding of one of those
# objects.
stderr = io.TextIOWrapper(stderr, encoding="utf-8")
stderr.write(msg)
traceback.print_exc(file=stderr)
stderr.flush()
for fd in xrange(3):
os.close(fd)
except:
# Handle all errors during the error-reporting process
# silently to ensure that the child terminates.
pass
# See comment above about making sure that we reach this line
# of code.
os._exit(1)
# we are now in parent process
if collectorEnabled:
gc.enable()
self.status = -1 # this records the exit status of the child
def _emptyLine(self, width):
return [(self.void, self._currentFormattingState())
for i in xrange(width)]
def _junkPath(self):
junkPath = self.mktemp()
with open(junkPath, 'wb') as junkFile:
for i in xrange(1024):
junkFile.write(intToBytes(i) + b'\n')
return junkPath
from atexit import register # 使用atexit.register()函数来告知脚本何时结束
from random import randrange
from twisted.python.compat import xrange
from threading import Thread, currentThread
from time import ctime, sleep
class CleanOutputSet(set):
def __str__(self):
return ', '.join(x for x in self)
# loops 是一个生成器
loops = (randrange(2, 5) for x in xrange(randrange(3, 7)))
remaining = CleanOutputSet()
def loop(nsec):
my_name = currentThread().name
remaining.add(my_name)
print('[{}] Started {} '.format(ctime(), my_name))
sleep(nsec)
remaining.remove(my_name)
print('[{}] Completed {} ({} secs)'.format(ctime(), my_name, nsec))
print('(remaining: {})'.format(remaining or 'NONE'))
def _main():
for pause in loops:
Thread(target=loop, args=(pause, )).start()
