def _parseAttributes(self, data): flags ,= struct.unpack('!L', data[:4]) attrs = {} data = data[4:] if flags & FILEXFER_ATTR_SIZE == FILEXFER_ATTR_SIZE: size ,= struct.unpack('!Q', data[:8]) attrs['size'] = size data = data[8:] if flags & FILEXFER_ATTR_OWNERGROUP == FILEXFER_ATTR_OWNERGROUP: uid, gid = struct.unpack('!2L', data[:8]) attrs['uid'] = uid attrs['gid'] = gid data = data[8:] if flags & FILEXFER_ATTR_PERMISSIONS == FILEXFER_ATTR_PERMISSIONS: perms ,= struct.unpack('!L', data[:4]) attrs['permissions'] = perms data = data[4:] if flags & FILEXFER_ATTR_ACMODTIME == FILEXFER_ATTR_ACMODTIME: atime, mtime = struct.unpack('!2L', data[:8]) attrs['atime'] = atime attrs['mtime'] = mtime data = data[8:] if flags & FILEXFER_ATTR_EXTENDED == FILEXFER_ATTR_EXTENDED: extended_count ,= struct.unpack('!L', data[:4]) data = data[4:] for i in xrange(extended_count): extended_type, data = getNS(data) extended_data, data = getNS(data) attrs['ext_%s' % nativeString(extended_type)] = extended_data return attrs, data
def render(): child = Elem() for _ in xrange(20): child = Elem([child]) root = TagLoader([child] * 10).load() out = BytesIO() flatten(None, root, out.write)
def _unzipIterChunkyTest(self, compression, chunksize, lower, upper): """ unzipIterChunky should unzip the given number of bytes per iteration. """ junk = ' '.join([str(random.random()) for n in xrange(1000)]) junkmd5 = md5(junk).hexdigest() tempdir = filepath.FilePath(self.mktemp()) tempdir.makedirs() zfpath = tempdir.child('bigfile.zip').path self._makebigfile(zfpath, compression, junk) uziter = zipstream.unzipIterChunky(zfpath, tempdir.path, chunksize=chunksize) r = uziter.next() # test that the number of chunks is in the right ballpark; # this could theoretically be any number but statistically it # should always be in this range approx = lower < r < upper self.assertTrue(approx) for r in uziter: pass self.assertEqual(r, 0) newmd5 = md5(tempdir.child("zipstreamjunk").open().read()).hexdigest() self.assertEqual(newmd5, junkmd5)
def _parseAttributes(self, data): flags, = struct.unpack('!L', data[:4]) attrs = {} data = data[4:] if flags & FILEXFER_ATTR_SIZE == FILEXFER_ATTR_SIZE: size, = struct.unpack('!Q', data[:8]) attrs['size'] = size data = data[8:] if flags & FILEXFER_ATTR_OWNERGROUP == FILEXFER_ATTR_OWNERGROUP: uid, gid = struct.unpack('!2L', data[:8]) attrs['uid'] = uid attrs['gid'] = gid data = data[8:] if flags & FILEXFER_ATTR_PERMISSIONS == FILEXFER_ATTR_PERMISSIONS: perms, = struct.unpack('!L', data[:4]) attrs['permissions'] = perms data = data[4:] if flags & FILEXFER_ATTR_ACMODTIME == FILEXFER_ATTR_ACMODTIME: atime, mtime = struct.unpack('!2L', data[:8]) attrs['atime'] = atime attrs['mtime'] = mtime data = data[8:] if flags & FILEXFER_ATTR_EXTENDED == FILEXFER_ATTR_EXTENDED: extended_count, = struct.unpack('!L', data[:4]) data = data[4:] for i in xrange(extended_count): extended_type, data = getNS(data) extended_data, data = getNS(data) attrs['ext_%s' % nativeString(extended_type)] = extended_data return attrs, data
def __init__(self, l, containerType): """ @param l: The list of object which may contain some not yet referenced objects. @param containerType: A type of container objects (e.g., C{tuple} or C{set}). """ NotKnown.__init__(self) self.containerType = containerType self.l = l self.locs = list(xrange(len(l))) for idx in xrange(len(l)): if not isinstance(l[idx], NotKnown): self.locs.remove(idx) else: l[idx].addDependant(self, idx) if not self.locs: self.resolveDependants(self.containerType(self.l))
def _resetSignalDisposition(self): # The Python interpreter ignores some signals, and our child # process will inherit that behaviour. To have a child process # that responds to signals normally, we need to reset our # child process's signal handling (just) after we fork and # before we execvpe. for signalnum in xrange(1, signal.NSIG): if signal.getsignal(signalnum) == signal.SIG_IGN: # Reset signal handling to the default signal.signal(signalnum, signal.SIG_DFL)
def ranges_to_sequence(ranges): """Iterate over individual items represented in a ranges list.""" for item in ranges: if isinstance(item, tuple): start, end = item if start > end: raise ValueError("Range error %d > %d", start, end) for item in xrange(start, end + 1): yield item else: yield item
def resumeProducing(self): """Called when the consumer attached to me runs out of buffer. """ # Go backwards over the list so we can remove indexes from it as we go for pageridx in xrange(len(self.pageProducers) - 1, -1, -1): pager = self.pageProducers[pageridx] pager.sendNextPage() if not pager.stillPaging(): del self.pageProducers[pageridx] if not self.pageProducers: self.transport.unregisterProducer()
def test_processCommandLineArguments(self): """ Arguments given to spawnProcess are passed to the child process as originally intended. """ us = FilePath(__file__).sibling(b"process_cli.py") args = [ b'hello', b'"', b' \t|<>^&', br'"\\"hello\\"', br'"foo\ bar baz\""' ] # Ensure that all non-NUL characters can be passed too. if _PY3: args.append("".join(map(chr, xrange(1, 255))).encode("utf8")) else: args.append("".join(map(chr, xrange(1, 255)))) reactor = self.buildReactor() def processFinished(finishedArgs): output, err, code = finishedArgs output = output.split(b'\0') # Drop the trailing \0. output.pop() self.assertEqual(args, output) def shutdown(result): reactor.stop() return result def spawnChild(): d = succeed(None) d.addCallback(lambda dummy: utils.getProcessOutputAndValue( pyExe, [us.path] + args, reactor=reactor)) d.addCallback(processFinished) d.addBoth(shutdown) reactor.callWhenRunning(spawnChild) self.runReactor(reactor)
def test_processCommandLineArguments(self): """ Arguments given to spawnProcess are passed to the child process as originally intended. """ us = b"twisted.internet.test.process_cli" args = [b'hello', b'"', b' \t|<>^&', br'"\\"hello\\"', br'"foo\ bar baz\""'] # Ensure that all non-NUL characters can be passed too. if _PY3: args.append("".join(map(chr, xrange(1,255))).encode("utf8")) else: args.append("".join(map(chr, xrange(1,255)))) reactor = self.buildReactor() def processFinished(finishedArgs): output, err, code = finishedArgs output = output.split(b'\0') # Drop the trailing \0. output.pop() self.assertEqual(args, output) def shutdown(result): reactor.stop() return result def spawnChild(): d = succeed(None) d.addCallback(lambda dummy: utils.getProcessOutputAndValue( pyExe, [b"-m", us] + args, env=properEnv, reactor=reactor)) d.addCallback(processFinished) d.addBoth(shutdown) reactor.callWhenRunning(spawnChild) self.runReactor(reactor)
def send_method_call(self, method, args=[], kwargs={}): """Send a L{MethodCall} command with the given arguments. If a response from the server is not received within C{self.timeout} seconds, the returned deferred will errback with a L{MethodCallError}. @param method: The name of the remote method to invoke. @param args: The positional arguments to pass to the remote method. @param kwargs: The keyword arguments to pass to the remote method. @return: A C{Deferred} firing with the return value of the method invoked on the remote object. If the remote method itself returns a deferred, we fire with the callback value of such deferred. """ arguments = bpickle.dumps((args, kwargs)) sequence = uuid4().int # As we send the method name to remote, we need bytes. method = method.encode("utf-8") # Split the given arguments in one or more chunks chunks = [arguments[i:i + self._chunk_size] for i in xrange(0, len(arguments), self._chunk_size)] result = Deferred() if len(chunks) > 1: # If we have N chunks, send the first N-1 as MethodCallChunk's for chunk in chunks[:-1]: def create_send_chunk(sequence, chunk): send_chunk = (lambda x: self._protocol.callRemote( MethodCallChunk, sequence=sequence, chunk=chunk)) return send_chunk result.addCallback(create_send_chunk(sequence, chunk)) def send_last_chunk(ignored): chunk = chunks[-1] return self._call_remote_with_timeout( MethodCall, sequence=sequence, method=method, arguments=chunk) result.addCallback(send_last_chunk) result.addCallback(lambda response: response["result"]) result.callback(None) return result
def eraseDisplay(self): self.lines = [self._emptyLine(self.width) for i in xrange(self.height)]
def _fork(self, path, uid, gid, executable, args, environment, **kwargs): """ Fork and then exec sub-process. @param path: the path where to run the new process. @type path: C{str} @param uid: if defined, the uid used to run the new process. @type uid: C{int} @param gid: if defined, the gid used to run the new process. @type gid: C{int} @param executable: the executable to run in a new process. @type executable: C{str} @param args: arguments used to create the new process. @type args: C{list}. @param environment: environment used for the new process. @type environment: C{dict}. @param kwargs: keyword arguments to L{_setupChild} method. """ collectorEnabled = gc.isenabled() gc.disable() try: self.pid = os.fork() except: # Still in the parent process if collectorEnabled: gc.enable() raise else: if self.pid == 0: # A return value of 0 from fork() indicates that we are now # executing in the child process. # Do not put *ANY* code outside the try block. The child # process must either exec or _exit. If it gets outside this # block (due to an exception that is not handled here, but # which might be handled higher up), there will be two copies # of the parent running in parallel, doing all kinds of damage. # After each change to this code, review it to make sure there # are no exit paths. try: # Stop debugging. If I am, I don't care anymore. sys.settrace(None) self._setupChild(**kwargs) self._execChild(path, uid, gid, executable, args, environment) except: # If there are errors, try to write something descriptive # to stderr before exiting. # The parent's stderr isn't *necessarily* fd 2 anymore, or # even still available; however, even libc assumes that # write(2, err) is a useful thing to attempt. try: stderr = os.fdopen(2, 'wb') msg = ("Upon execvpe {0} {1} in environment id {2}" "\n:").format(executable, str(args), id(environment)) if _PY3: # On Python 3, print_exc takes a text stream, but # on Python 2 it still takes a byte stream. So on # Python 3 we will wrap up the byte stream returned # by os.fdopen using TextIOWrapper. # We hard-code UTF-8 as the encoding here, rather # than looking at something like # getfilesystemencoding() or sys.stderr.encoding, # because we want an encoding that will be able to # encode the full range of code points. We are # (most likely) talking to the parent process on # the other end of this pipe and not the filesystem # or the original sys.stderr, so there's no point # in trying to match the encoding of one of those # objects. stderr = io.TextIOWrapper(stderr, encoding="utf-8") stderr.write(msg) traceback.print_exc(file=stderr) stderr.flush() for fd in xrange(3): os.close(fd) except: # Handle all errors during the error-reporting process # silently to ensure that the child terminates. pass # See comment above about making sure that we reach this line # of code. os._exit(1) # we are now in parent process if collectorEnabled: gc.enable() self.status = -1 # this records the exit status of the child
def _fork(self, path, uid, gid, executable, args, environment, **kwargs): """ Fork and then exec sub-process. @param path: the path where to run the new process. @type path: C{str} @param uid: if defined, the uid used to run the new process. @type uid: C{int} @param gid: if defined, the gid used to run the new process. @type gid: C{int} @param executable: the executable to run in a new process. @type executable: C{str} @param args: arguments used to create the new process. @type args: C{list}. @param environment: environment used for the new process. @type environment: C{dict}. @param kwargs: keyword arguments to L{_setupChild} method. """ collectorEnabled = gc.isenabled() gc.disable() try: self.pid = os.fork() except: # Still in the parent process if collectorEnabled: gc.enable() raise else: if self.pid == 0: # pid is 0 in the child process # do not put *ANY* code outside the try block. The child process # must either exec or _exit. If it gets outside this block (due # to an exception that is not handled here, but which might be # handled higher up), there will be two copies of the parent # running in parallel, doing all kinds of damage. # After each change to this code, review it to make sure there # are no exit paths. try: # Stop debugging. If I am, I don't care anymore. sys.settrace(None) self._setupChild(**kwargs) self._execChild( path, uid, gid, executable, args, environment) except: # If there are errors, bail and try to write something # descriptive to stderr. # XXX: The parent's stderr isn't necessarily fd 2 anymore, or # even still available # XXXX: however even libc assumes write(2, err) is a useful # thing to attempt try: stderr = os.fdopen(2, 'wb') msg = ("Upon execvpe {0} {1} in environment id {2}" "\n:").format(executable, str(args), id(environment)) tb = NativeStringIO() traceback.print_exc(file=tb) tb = tb.getvalue() if _PY3: msg = msg.encode(sys.getfilesystemencoding()) tb = tb.encode(sys.getfilesystemencoding()) stderr.write(msg) stderr.write(tb) stderr.flush() for fd in xrange(3): os.close(fd) except: pass # make *sure* the child terminates # Did you read the comment about not adding code here? os._exit(1) # we are now in parent process if collectorEnabled: gc.enable() self.status = -1 # this records the exit status of the child
def _emptyLine(self, width): return [(self.void, self._currentFormattingState()) for i in xrange(width)]
def _junkPath(self): junkPath = self.mktemp() with open(junkPath, 'wb') as junkFile: for i in xrange(1024): junkFile.write(intToBytes(i) + b'\n') return junkPath
from atexit import register # 使用atexit.register()函数来告知脚本何时结束 from random import randrange from twisted.python.compat import xrange from threading import Thread, currentThread from time import ctime, sleep class CleanOutputSet(set): def __str__(self): return ', '.join(x for x in self) # loops 是一个生成器 loops = (randrange(2, 5) for x in xrange(randrange(3, 7))) remaining = CleanOutputSet() def loop(nsec): my_name = currentThread().name remaining.add(my_name) print('[{}] Started {} '.format(ctime(), my_name)) sleep(nsec) remaining.remove(my_name) print('[{}] Completed {} ({} secs)'.format(ctime(), my_name, nsec)) print('(remaining: {})'.format(remaining or 'NONE')) def _main(): for pause in loops: Thread(target=loop, args=(pause, )).start()