def _http_get(self, path: str) -> SpooledTemporaryFile: url = urllib.parse.urljoin(self.repo_url.rstrip("/") + "/", path) logger.debug("Fetching %s", url) response = self._session.get(url, headers=HEADERS) buffer = SpooledTemporaryFile(max_size=100 * 1024 * 1024) for chunk in response.iter_content(chunk_size=10 * 1024 * 1024): buffer.write(chunk) buffer.flush() buffer.seek(0) return buffer
class RemoteFileBuffer(object): """File-like object providing buffer for local file operations. Instances of this class manage a local tempfile buffer corresponding to the contents of a remote file. All reads and writes happen locally, with the content being copied to the remote file only on flush() or close(). Instances of this class are returned by S3FS.open, but it is desgined to be usable by any FS subclass that manages remote files. """ def __init__(self,fs,path,mode): self.file = TempFile() self.fs = fs self.path = path self.mode = mode def __del__(self): if not self.closed: self.close() # This is lifted straight from the stdlib's tempfile.py def __getattr__(self,name): file = self.__dict__['file'] a = getattr(file, name) if not issubclass(type(a), type(0)): setattr(self, name, a) return a def __enter__(self): self.file.__enter__() return self def __exit__(self,exc,value,tb): self.close() return False def __iter__(self): return iter(self.file) def flush(self): self.file.flush() if "w" in self.mode or "a" in self.mode or "+" in self.mode: pos = self.file.tell() self.file.seek(0) self.fs.setcontents(self.path,self.file) self.file.seek(pos) def close(self): if "w" in self.mode or "a" in self.mode or "+" in self.mode: self.file.seek(0) self.fs.setcontents(self.path,self.file) self.file.close()
def fetch_pack_from_origin( self, origin_url: str, base_repo: RepoRepresentation, do_activity: Callable[[bytes], None], ) -> FetchPackReturn: """Fetch a pack from the origin""" pack_buffer = SpooledTemporaryFile(max_size=self.temp_file_cutoff) transport_url = origin_url logger.debug("Transport url to communicate with server: %s", transport_url) client, path = dulwich.client.get_transport_and_path( transport_url, thin_packs=False ) logger.debug("Client %s to fetch pack at %s", client, path) size_limit = self.pack_size_bytes def do_pack(data: bytes) -> None: cur_size = pack_buffer.tell() would_write = len(data) if cur_size + would_write > size_limit: raise IOError( f"Pack file too big for repository {origin_url}, " f"limit is {size_limit} bytes, current size is {cur_size}, " f"would write {would_write}" ) pack_buffer.write(data) pack_result = client.fetch_pack( path, base_repo.determine_wants, base_repo.graph_walker(), do_pack, progress=do_activity, ) remote_refs = pack_result.refs or {} symbolic_refs = pack_result.symrefs or {} pack_buffer.flush() pack_size = pack_buffer.tell() pack_buffer.seek(0) logger.debug("fetched_pack_size=%s", pack_size) # check if repository only supports git dumb transfer protocol, # fetched pack file will be empty in that case as dulwich do # not support it and do not fetch any refs self.dumb = transport_url.startswith("http") and getattr(client, "dumb", False) return FetchPackReturn( remote_refs=utils.filter_refs(remote_refs), symbolic_refs=utils.filter_refs(symbolic_refs), pack_buffer=pack_buffer, pack_size=pack_size, )
def fetch_media(self, url, partial_fetch=False): """Retrieves a given media object from a remote (HTTP) location and returns the content-type and a file-like object containing the media content. The file-like object is a temporary file that - depending on the size - lives in memory or on disk. Once the file is closed, the contents are removed from storage. :param url: the URL of the media asset. :type url: str. :param partial_fetch: determines if the the complete file should be fetched, or if only the first 2 MB should be retrieved. This feature is used to prevent complete retrieval of large a/v material. :type partial_fetch: bool. :returns: a tuple with the ``content-type``, ``content-lenght`` and a file-like object containing the media content. The value of ``content-length`` will be ``None`` in case a partial fetch is requested and ``content-length`` is not returned by the remote server. """ http_resp = self.http_session.get(url, stream=True, timeout=(60, 120)) http_resp.raise_for_status() if not os.path.exists(TEMP_DIR_PATH): log.debug('Creating temp directory %s' % TEMP_DIR_PATH) os.makedirs(TEMP_DIR_PATH) # Create a temporary file to store the media item, write the file # to disk if it is larger than 1 MB. media_file = SpooledTemporaryFile(max_size=1024 * 1024, prefix='oad_m_', suffix='.tmp', dir=TEMP_DIR_PATH) # When a partial fetch is requested, request up to two MB partial_target_size = 1024 * 1024 * 2 content_length = http_resp.headers.get('content-length') if content_length and int(content_length) < partial_target_size: partial_target_size = int(content_length) retrieved_bytes = 0 for chunk in http_resp.iter_content(chunk_size=512 * 1024): if chunk: # filter out keep-alive chunks media_file.write(chunk) retrieved_bytes += len(chunk) if partial_fetch and retrieved_bytes >= partial_target_size: break media_file.flush() log.debug('Fetched media item %s [%s/%s]' % (url, retrieved_bytes, content_length)) # If the server doens't provide a content-length and this isn't # a partial fetch, determine the size by looking at the retrieved # content if not content_length and not partial_fetch: media_file.seek(0, 2) content_length = media_file.tell() return (http_resp.headers.get('content-type'), content_length, media_file)
class RecvWindow(object): """ Receiver windows receive data on a socket to fill their buffer. A window instance has the following attributes: name Name of the window. The name of all windows is shown by '> win show' in command line interface. It's usually the name of the plugin appended by the arguments passed for execution (e.g. ps.plg -A -t curesec.com) status indicates the state of the plugin (e.g. RUNNING or FINISHED) req_t request tuple that triggered plugin execution. It's used to verify incoming packets plg_id id of the plugin rid request id of the packet triggering execution wid the id of the window. Derived from a class variable which is increased after window creation. The window id is not fixed. So every new client session, the windows might have a new id. session the actual client session is_used flag, that indicates, whether the windows is used by user (via '> win show <id>') """ #: command to close window CMD_CLOSE = "close" #: window id, increased for every new window ID = 1 def __init__(self, name, req_t, session): logger.debug("creating window %s", name) self.name = name self.status = RUNNING self.req_t = req_t self.sid = req_t[1] self.rid = req_t[2] self.plg_id = req_t[4] self.wid = RecvWindow.ID self.session = session self.is_used = False self.closed = False self._buffer = SpooledTemporaryFile(max_size="8096", mode="wr+b") RecvWindow.ID += 1 def close(self): self.closed = True print("closing window %d" % self.wid) def set_state(self, input_code): state = None try: state = STATE[input_code + 1] except (IndexError, TypeError) as e: logger.warning('Could not get window state for code %d: %s', input_code, e) logger.exception(e) if not state: state = "Unknown" if self.is_used and state in (FINISHED, ERROR, KILLED): self.is_used = False code = "(%s)" % str(input_code) self.status = "{state} {code}".format(state=state, code=code) logger.debug("new window(id:%s) state: %s", self.wid, self.status) def _write_to_output(self, data): # if no more data is coming, set window state to # finished and break if we have a non-interactive # window (otherwise we need to wait for the user # to enter further input). #if not data: # self.status = FINISHED # TODO: that seems a bit heuristic to me ... # ok, we have data, so verify 'em and print it. this is # done by unpack try: actual_output = self._unpack(data) if actual_output: sys.stdout.write(actual_output) sys.stdout.flush() self._buffer.write(actual_output) self._buffer.flush() except errors.ClientError as e: if e.request: logger.warning('Request: %s', conn.ccdlib.pkt2str(e.request)) if e.response: logger.warning('Response: %s', conn.ccdlib.pkt2str(e.response)) logger.debug("received invalid packet:'%s'", e) logger.exception(e) except Exception as e: logger.debug("received invalid packet:'%s'", e) logger.exception(e) def _unpack(self, resp_t): #if not resp_t[0] == cnnm.ccdlib.OP_SUCCESS: # raise Exception("received packet that indicates a failure(%s)!", # hex(resp_t[0])) ## sid #if not resp_t[1] == self.req_t[1]: # raise Exception("received packet with invalid session id(%s)!", # resp_t[1]) ## rid #if not resp_t[2] == self.req_t[2]: # raise Exception("received packet with invalid rid(%s)!", # resp_t[2]) ## plg_id #if not resp_t[4] == self.req_t[4]: # raise Exception("received packet with invalid plugin id(%s)!", # resp_t[4]) # method indicates a plugin's state change. if resp_t[6] == conn.ccdlib.MTH_TERMINATE: state = 2 try: state = resp_t[-1]["code"] except (KeyError, IndexError): raise errors.InvalidServerResponse( message="No state in terminate response payload", request=self.req_t, response=resp_t) self.set_state(state) return elif resp_t[6] == conn.ccdlib.MTH_INPUT: return "" # plugin gives outpout elif not resp_t[6] == conn.ccdlib.MTH_OUTPUT: try: MTH_str = conn.ccdlib.rev_MTH_map[resp_t[6]] except: MTH_str = repr(resp_t[6]) raise errors.InvalidServerResponse( message= "Invalid method %s, expected MTH_OUTPUT, _TERMINATE or _INPUT" % MTH_str, request=self.req_t, response=resp_t) return resp_t[-1] def _pack(self, data): req_t = list(self.req_t) req_t[ 0] = conn.ccdlib.OP_PLUGIN # indicate operation concerning plugin req_t[6] = conn.ccdlib.MTH_INPUT # indicate plugin input is coming req_t[-1] = data # payload # if no input to commit, raise EmptyException if not req_t[-1]: raise conn.ccdlib.EmptyException return tuple(req_t) def use(self): # to get the plugin's output, we need to register rid = self.req_t[2] plg_id = self.req_t[4] pld = comm.sendpacket(self.session, op=conn.ccdlib.OP_REGISTER, plg=plg_id, pld=dict(rid=rid))[-1] interactive = pld["interactive"] logger.debug("successfully registered to plugin. have fun!") if interactive: print("Enter '%s' to leave interactive mode." % RecvWindow.CMD_CLOSE) sock = self.session.sock # write content that is already buffered for line in self._buffer: sys.stdout.write(line) # start window interaction self.is_used = True while self.is_used: try: r, _, _ = select.select([sys.stdin, sock], [], []) except KeyboardInterrupt: print("catched keyboard interrupt. closing window.") self.is_used = False break # if there is something to print to window if sock in r: # read data, that should be written to stdout. # the data is sent by the ccd resp_t = None try: resp_t = conn.ccdlib.recv(sock) #resp_t = comm.wait_for_response(self.session, self.req_t) except Exception as e: logger.error("Exception while receiving data: %s", e) logger.exception(e) self.is_used = False break if resp_t: self._write_to_output(resp_t) # there is something to read form window console if sys.stdin in r: if not interactive: # any keystroke sends active plugin to background self.is_used = False break # read the user's input to be sent to the ccd plugin #data = sys.stdin.readline() data = '' try: data = os.read(sys.stdin.fileno(), 1024) except Exception, e: logger.error("Error reading from stdin: '%s'!", e) logger.exception(e) continue logger.debug('all read on stdin \'%s\'', data) # catch command to close the interactive window if data.rstrip() == RecvWindow.CMD_CLOSE: self.is_used = False break # the actual write of the user's input actual_packet = self._pack(data) try: conn.ccdlib.send(sock, actual_packet) #comm.wait_for_response(self.session, actual_packet) except Exception as e: logger.error("Failed to send user input to plugin!:'%s'", e) logger.exception(e) try: comm.sendpacket(self.session, op=conn.ccdlib.OP_UNREGISTER, plg=plg_id, pld=dict(rid=rid)) except KeyboardInterrupt: print("catched keyboard interrupt. closing window.") except Exception as e: logger.error("Failed to unregister properly:'%s'!", e) logger.exception(e)
def flush(self): """Flush the data to the server.""" SpooledTemporaryFile.flush(self) self.commit()
def flush(self): # flush data to server SpooledTemporaryFile.flush(self) self.commit()
def fetch_media(self, url, partial_fetch=False): """Retrieves a given media object from a remote (HTTP) location and returns the content-type and a file-like object containing the media content. The file-like object is a temporary file that - depending on the size - lives in memory or on disk. Once the file is closed, the contents are removed from storage. :param url: the URL of the media asset. :type url: str. :param partial_fetch: determines if the the complete file should be fetched, or if only the first 2 MB should be retrieved. This feature is used to prevent complete retrieval of large a/v material. :type partial_fetch: bool. :returns: a tuple with the ``content-type``, ``content-lenght`` and a file-like object containing the media content. The value of ``content-length`` will be ``None`` in case a partial fetch is requested and ``content-length`` is not returned by the remote server. """ http_resp = self.http_session.get(url, stream=True, timeout=(60, 120)) http_resp.raise_for_status() if not os.path.exists(TEMP_DIR_PATH): log.debug('Creating temp directory %s' % TEMP_DIR_PATH) os.makedirs(TEMP_DIR_PATH) # Create a temporary file to store the media item, write the file # to disk if it is larger than 1 MB. media_file = SpooledTemporaryFile(max_size=1024*1024, prefix='ocd_m_', suffix='.tmp', dir=TEMP_DIR_PATH) # When a partial fetch is requested, request up to two MB partial_target_size = 1024*1024*2 content_length = http_resp.headers.get('content-length') if content_length and int(content_length) < partial_target_size: partial_target_size = int(content_length) retrieved_bytes = 0 for chunk in http_resp.iter_content(chunk_size=512*1024): if chunk: # filter out keep-alive chunks media_file.write(chunk) retrieved_bytes += len(chunk) if partial_fetch and retrieved_bytes >= partial_target_size: break media_file.flush() log.debug('Fetched media item %s [%s/%s]' % (url, retrieved_bytes, content_length)) # If the server doens't provide a content-length and this isn't # a partial fetch, determine the size by looking at the retrieved # content if not content_length and not partial_fetch: media_file.seek(0, 2) content_length = media_file.tell() return ( http_resp.headers.get('content-type'), content_length, media_file )
class Buffer(FileWrapper): """Class implementing buffereing of input and output streams. This class uses a separate buffer file to hold the contents of the underlying file while they are being manipulated. As data is read it is duplicated into the buffer, and data is written from the buffer back to the file on close. """ def __init__(self,fileobj,mode=None,max_size_in_memory=1024*8): """Buffered file wrapper constructor.""" self._buffer = SpooledTemporaryFile(max_size=max_size_in_memory) self._in_eof = False self._in_pos = 0 super(Buffer,self).__init__(fileobj,mode) def _buffer_chunks(self): chunk = self._buffer.read(16*1024) if chunk == "": yield chunk else: while chunk != "": yield chunk chunk = self._buffer.read(16*1024) def _write_out_buffer(self): if self._check_mode("r"): self._read_rest() if "a" in self.mode: self._buffer.seek(self._in_pos) self._fileobj.seek(self._in_pos) else: self._fileobj.seek(0) self._buffer.seek(0) else: self._buffer.seek(0) for chunk in self._buffer_chunks(): self._fileobj.write(chunk) def flush(self): # flush the buffer; we only write to the underlying file on close self._buffer.flush() def close(self): if self.closed: return if self._check_mode("w"): self._write_out_buffer() super(Buffer,self).close() self._buffer.close() def _read(self,sizehint=-1): # First return any data available from the buffer. # Since we don't flush the buffer after every write, certain OSes # (guess which!) will happy read junk data from the end of it. # Instead, we explicitly read only up to self._in_pos. if not self._in_eof: buffered_size = self._in_pos - self._buffer.tell() if sizehint >= 0: buffered_size = min(sizehint,buffered_size) else: buffered_size = sizehint data = self._buffer.read(buffered_size) if data != "": return data # Then look for more data in the underlying file if self._in_eof: return None data = self._fileobj.read(sizehint) self._in_pos += len(data) self._buffer.write(data) if sizehint < 0 or len(data) < sizehint: self._in_eof = True self._buffer.flush() return data def _write(self,data,flushing=False): self._buffer.write(data) if self._check_mode("r") and not self._in_eof: diff = self._buffer.tell() - self._in_pos if diff > 0: junk = self._fileobj.read(diff) self._in_pos += len(junk) if len(junk) < diff: self._in_eof = True self._buffer.flush() def _seek(self,offset,whence): # Ensure we've read enough to simply do the seek on the buffer if self._check_mode("r") and not self._in_eof: if whence == 0: if offset > self._in_pos: self._read_rest() if whence == 1: if self._buffer.tell() + offset > self._in_pos: self._read_rest() if whence == 2: self._read_rest() # Then just do it on the buffer... self._buffer.seek(offset,whence) def _tell(self): return self._buffer.tell() def _read_rest(self): """Read the rest of the input stream.""" if self._in_eof: return pos = self._buffer.tell() self._buffer.seek(0,2) data = self._fileobj.read(self._bufsize) while data: self._in_pos += len(data) self._buffer.write(data) data = self._fileobj.read(self._bufsize) self._in_eof = True self._buffer.flush() self._buffer.seek(pos)
class Buffer(FileWrapper): """Class implementing buffering of input and output streams. This class uses a separate buffer file to hold the contents of the underlying file while they are being manipulated. As data is read it is duplicated into the buffer, and data is written from the buffer back to the file on close. """ def __init__(self, fileobj, mode=None, max_size_in_memory=1024 * 8): """Buffered file wrapper constructor.""" self._buffer = SpooledTemporaryFile(max_size=max_size_in_memory) self._in_eof = False self._in_pos = 0 self._was_truncated = False super(Buffer, self).__init__(fileobj, mode) def _buffer_size(self): try: return len(self._buffer.file.getvalue()) except AttributeError: return os.fstat(self._buffer.fileno()).st_size def _buffer_chunks(self): chunk = self._buffer.read(16 * 1024) if chunk == "": yield chunk else: while chunk != "": yield chunk chunk = self._buffer.read(16 * 1024) def _write_out_buffer(self): if self._check_mode("r"): self._read_rest() if "a" in self.mode: self._buffer.seek(self._in_pos) self._fileobj.seek(self._in_pos) else: self._fileobj.seek(0) self._buffer.seek(0) else: self._buffer.seek(0) if self._was_truncated: self._fileobj.truncate(0) self._was_truncated = False for chunk in self._buffer_chunks(): self._fileobj.write(chunk) def flush(self): # flush the buffer; we only write to the underlying file on close self._buffer.flush() def close(self): if self.closed: return if self._check_mode("w"): self._write_out_buffer() super(Buffer, self).close() self._buffer.close() def _read(self, sizehint=-1): # First return any data available from the buffer. # Since we don't flush the buffer after every write, certain OSes # (guess which!) will happily read junk data from the end of it. # Instead, we explicitly read only up to self._in_pos. if not self._in_eof: buffered_size = self._in_pos - self._buffer.tell() if sizehint >= 0: buffered_size = min(sizehint, buffered_size) else: buffered_size = sizehint data = self._buffer.read(buffered_size) if data != "": return data # Then look for more data in the underlying file if self._in_eof: return None data = self._fileobj.read(sizehint) self._in_pos += len(data) self._buffer.write(data) if sizehint < 0 or len(data) < sizehint: self._in_eof = True self._buffer.flush() return data def _write(self, data, flushing=False): self._buffer.write(data) if self._check_mode("r") and not self._in_eof: diff = self._buffer.tell() - self._in_pos if diff > 0: junk = self._fileobj.read(diff) self._in_pos += len(junk) if len(junk) < diff: self._in_eof = True self._buffer.flush() def _seek(self, offset, whence): # Ensure we've read enough to simply do the seek on the buffer if self._check_mode("r") and not self._in_eof: if whence == 0: if offset > self._in_pos: self._read_rest() if whence == 1: if self._buffer.tell() + offset > self._in_pos: self._read_rest() if whence == 2: self._read_rest() # Then just do it on the buffer... self._buffer.seek(offset, whence) def _tell(self): return self._buffer.tell() def _truncate(self, size): if self._check_mode("r") and not self._in_eof: if size > self._in_pos: self._read_rest() self._in_eof = True try: self._buffer.truncate(size) except TypeError: et, ev, tb = sys.exc_info() # SpooledTemporaryFile.truncate() doesn't accept size paramter. try: self._buffer._file.truncate(size) except Exception: raise et, ev, tb # StringIO objects don't truncate to larger size correctly. if hasattr(self._buffer, "_file"): _file = self._buffer._file if hasattr(_file, "getvalue"): if len(_file.getvalue()) != size: curpos = _file.tell() _file.seek(0, 2) _file.write("\x00" * (size - len(_file.getvalue()))) _file.seek(curpos) self._was_truncated = True def _read_rest(self): """Read the rest of the input stream.""" if self._in_eof: return pos = self._buffer.tell() self._buffer.seek(0, 2) data = self._fileobj.read(self._bufsize) while data: self._in_pos += len(data) self._buffer.write(data) data = self._fileobj.read(self._bufsize) self._in_eof = True self._buffer.flush() self._buffer.seek(pos)
def flush(self): SpooledTemporaryFile.flush(self) self.commit()
class RecvWindow(object): """ Receiver windows receive data on a socket to fill their buffer. A window instance has the following attributes: name Name of the window. The name of all windows is shown by '> win show' in command line interface. It's usually the name of the plugin appended by the arguments passed for execution (e.g. ps.plg -A -t curesec.com) status indicates the state of the plugin (e.g. RUNNING or FINISHED) req_t request tuple that triggered plugin execution. It's used to verify incoming packets plg_id id of the plugin rid request id of the packet triggering execution wid the id of the window. Derived from a class variable which is increased after window creation. The window id is not fixed. So every new client session, the windows might have a new id. session the actual client session is_used flag, that indicates, whether the windows is used by user (via '> win show <id>') """ #: command to close window CMD_CLOSE = "close" #: window id, increased for every new window ID = 1 def __init__(self, name, req_t, session): logger.debug("creating window %s", name) self.name = name self.status = RUNNING self.req_t = req_t self.sid = req_t[1] self.rid = req_t[2] self.plg_id = req_t[4] self.wid = RecvWindow.ID self.session = session self.is_used = False self.closed = False self._buffer = SpooledTemporaryFile(max_size="8096", mode="wr+b") RecvWindow.ID += 1 def close(self): self.closed = True print("closing window %d" % self.wid) def set_state(self, input_code): state = None try: state = STATE[input_code + 1] except (IndexError, TypeError) as e: logger.warning('Could not get window state for code %d: %s', input_code, e) logger.exception(e) if not state: state = "Unknown" if self.is_used and state in (FINISHED, ERROR, KILLED): self.is_used = False code = "(%s)" % str(input_code) self.status = "{state} {code}".format(state=state, code=code) logger.debug("new window(id:%s) state: %s", self.wid, self.status) def _write_to_output(self, data): # if no more data is coming, set window state to # finished and break if we have a non-interactive # window (otherwise we need to wait for the user # to enter further input). #if not data: # self.status = FINISHED # TODO: that seems a bit heuristic to me ... # ok, we have data, so verify 'em and print it. this is # done by unpack try: actual_output = self._unpack(data) if actual_output: sys.stdout.write(actual_output) sys.stdout.flush() self._buffer.write(actual_output) self._buffer.flush() except errors.ClientError as e: if e.request: logger.warning('Request: %s', conn.ccdlib.pkt2str(e.request)) if e.response: logger.warning('Response: %s', conn.ccdlib.pkt2str(e.response)) logger.debug("received invalid packet:'%s'", e) logger.exception(e) except Exception as e: logger.debug("received invalid packet:'%s'", e) logger.exception(e) def _unpack(self, resp_t): #if not resp_t[0] == cnnm.ccdlib.OP_SUCCESS: # raise Exception("received packet that indicates a failure(%s)!", # hex(resp_t[0])) ## sid #if not resp_t[1] == self.req_t[1]: # raise Exception("received packet with invalid session id(%s)!", # resp_t[1]) ## rid #if not resp_t[2] == self.req_t[2]: # raise Exception("received packet with invalid rid(%s)!", # resp_t[2]) ## plg_id #if not resp_t[4] == self.req_t[4]: # raise Exception("received packet with invalid plugin id(%s)!", # resp_t[4]) # method indicates a plugin's state change. if resp_t[6] == conn.ccdlib.MTH_TERMINATE: state = 2 try: state = resp_t[-1]["code"] except (KeyError, IndexError): raise errors.InvalidServerResponse( message="No state in terminate response payload", request=self.req_t, response=resp_t) self.set_state(state) return elif resp_t[6] == conn.ccdlib.MTH_INPUT: return "" # plugin gives outpout elif not resp_t[6] == conn.ccdlib.MTH_OUTPUT: try: MTH_str = conn.ccdlib.rev_MTH_map[resp_t[6]] except: MTH_str = repr(resp_t[6]) raise errors.InvalidServerResponse( message="Invalid method %s, expected MTH_OUTPUT, _TERMINATE or _INPUT" % MTH_str, request=self.req_t, response=resp_t) return resp_t[-1] def _pack(self, data): req_t = list(self.req_t) req_t[0] = conn.ccdlib.OP_PLUGIN # indicate operation concerning plugin req_t[6] = conn.ccdlib.MTH_INPUT # indicate plugin input is coming req_t[-1] = data # payload # if no input to commit, raise EmptyException if not req_t[-1]: raise conn.ccdlib.EmptyException return tuple(req_t) def use(self): # to get the plugin's output, we need to register rid = self.req_t[2] plg_id = self.req_t[4] pld = comm.sendpacket(self.session, op=conn.ccdlib.OP_REGISTER, plg=plg_id, pld=dict(rid=rid))[-1] interactive = pld["interactive"] logger.debug("successfully registered to plugin. have fun!") if interactive: print("Enter '%s' to leave interactive mode." % RecvWindow.CMD_CLOSE) sock = self.session.sock # write content that is already buffered for line in self._buffer: sys.stdout.write(line) # start window interaction self.is_used = True while self.is_used: try: r, _, _ = select.select([sys.stdin, sock], [], []) except KeyboardInterrupt: print("catched keyboard interrupt. closing window.") self.is_used = False break # if there is something to print to window if sock in r: # read data, that should be written to stdout. # the data is sent by the ccd resp_t = None try: resp_t = conn.ccdlib.recv(sock) #resp_t = comm.wait_for_response(self.session, self.req_t) except Exception as e: logger.error("Exception while receiving data: %s", e) logger.exception(e) self.is_used = False break if resp_t: self._write_to_output(resp_t) # there is something to read form window console if sys.stdin in r: if not interactive: # any keystroke sends active plugin to background self.is_used = False break # read the user's input to be sent to the ccd plugin #data = sys.stdin.readline() data = '' try: data = os.read(sys.stdin.fileno(), 1024) except Exception, e: logger.error("Error reading from stdin: '%s'!", e) logger.exception(e) continue logger.debug('all read on stdin \'%s\'', data) # catch command to close the interactive window if data.rstrip() == RecvWindow.CMD_CLOSE: self.is_used = False break # the actual write of the user's input actual_packet = self._pack(data) try: conn.ccdlib.send(sock, actual_packet) #comm.wait_for_response(self.session, actual_packet) except Exception as e: logger.error("Failed to send user input to plugin!:'%s'", e) logger.exception(e) try: comm.sendpacket(self.session, op=conn.ccdlib.OP_UNREGISTER, plg=plg_id, pld=dict(rid=rid)) except KeyboardInterrupt: print("catched keyboard interrupt. closing window.") except Exception as e: logger.error("Failed to unregister properly:'%s'!", e) logger.exception(e)
def exposed_cache_report(self, exporter_id, options): exporter = self._sa._manager.create_exporter(exporter_id, options) exported_stream = SpooledTemporaryFile(max_size=102400, mode='w+b') exporter.dump_from_session_manager(self._sa._manager, exported_stream) exported_stream.flush() return exported_stream