def _do_execute_direct(self, code):
shell = builtins.__xonsh_shell__
env = builtins.__xonsh_env__
out = io.StringIO()
err = io.StringIO()
enc = env.get('XONSH_ENCODING')
out = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t',
encoding=enc, newline='\n')
err = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t',
encoding=enc, newline='\n')
try:
with redirect_stdout(out), redirect_stderr(err), \
swap(builtins, '__xonsh_stdout_uncaptured__', out), \
swap(builtins, '__xonsh_stderr_uncaptured__', err), \
env.swap({'XONSH_STORE_STDOUT': False}):
shell.default(code)
interrupted = False
except KeyboardInterrupt:
interrupted = True
output, error = '', ''
if out.tell() > 0:
out.seek(0)
output = out.read()
if err.tell() > 0:
err.seek(0)
error = err.read()
out.close()
err.close()
return output, error, interrupted
def do_execute(self, code, silent, store_history=True, user_expressions=None,
allow_stdin=False):
"""Execute user code."""
if len(code.strip()) == 0:
return {'status': 'ok', 'execution_count': self.execution_count,
'payload': [], 'user_expressions': {}}
env = builtins.__xonsh_env__
shell = builtins.__xonsh_shell__
hist = builtins.__xonsh_history__
enc = env.get('XONSH_ENCODING')
out = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t',
encoding=enc, newline='\n')
err = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t',
encoding=enc, newline='\n')
try:
with redirect_stdout(out), redirect_stderr(err), \
swap(builtins, '__xonsh_stdout_uncaptured__', out), \
swap(builtins, '__xonsh_stderr_uncaptured__', err), \
env.swap({'XONSH_STORE_STDOUT': False}):
shell.default(code)
interrupted = False
except KeyboardInterrupt:
interrupted = True
if not silent: # stdout response
if out.tell() > 0:
out.seek(0)
self._respond_in_chunks('stdout', out.read())
if err.tell() > 0:
err.seek(0)
self._respond_in_chunks('stderr', err.read())
if hasattr(builtins, '_') and builtins._ is not None:
# rely on sys.displayhook functionality
self._respond_in_chunks('stdout', pformat(builtins._))
builtins._ = None
if len(hist) > 0 and out.tell() == 0 and err.tell() == 0:
self._respond_in_chunks('stdout', hist.outs[-1])
out.close()
err.close()
if interrupted:
return {'status': 'abort', 'execution_count': self.execution_count}
rtn = 0 if len(hist) == 0 else hist.rtns[-1]
if 0 < rtn:
message = {'status': 'error', 'execution_count': self.execution_count,
'ename': '', 'evalue': str(rtn), 'traceback': []}
else:
message = {'status': 'ok', 'execution_count': self.execution_count,
'payload': [], 'user_expressions': {}}
return message
def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False):
"""Execute user code."""
if len(code.strip()) == 0:
return {"status": "ok", "execution_count": self.execution_count, "payload": [], "user_expressions": {}}
env = builtins.__xonsh_env__
shell = builtins.__xonsh_shell__
hist = builtins.__xonsh_history__
enc = env.get("XONSH_ENCODING")
out = SpooledTemporaryFile(max_size=MAX_SIZE, mode="w+t", encoding=enc, newline="\n")
err = SpooledTemporaryFile(max_size=MAX_SIZE, mode="w+t", encoding=enc, newline="\n")
try:
with redirect_stdout(out), redirect_stderr(err), swap(builtins, "__xonsh_stdout_uncaptured__", out), swap(
builtins, "__xonsh_stderr_uncaptured__", err
), env.swap({"XONSH_STORE_STDOUT": False}):
shell.default(code)
interrupted = False
except KeyboardInterrupt:
interrupted = True
if not silent: # stdout response
if out.tell() > 0:
out.seek(0)
self._respond_in_chunks("stdout", out.read())
if err.tell() > 0:
err.seek(0)
self._respond_in_chunks("stderr", err.read())
if hasattr(builtins, "_") and builtins._ is not None:
# rely on sys.displayhook functionality
self._respond_in_chunks("stdout", pformat(builtins._))
builtins._ = None
if len(hist) > 0 and out.tell() == 0 and err.tell() == 0:
self._respond_in_chunks("stdout", hist.outs[-1])
out.close()
err.close()
if interrupted:
return {"status": "abort", "execution_count": self.execution_count}
rtn = 0 if len(hist) == 0 else hist.rtns[-1]
if 0 < rtn:
message = {
"status": "error",
"execution_count": self.execution_count,
"ename": "",
"evalue": str(rtn),
"traceback": [],
}
else:
message = {"status": "ok", "execution_count": self.execution_count, "payload": [], "user_expressions": {}}
return message
def graph(self):
if self.code is '':
return None
stdin = SpooledTemporaryFile()
stdout = SpooledTemporaryFile()
stdin.write('@startuml\n')
stdin.write(self.code)
stdin.write('@enduml\n')
stdin.seek(0)
args = [
self.java,
'-jar',
self.jar,
'-p',
'-tdot',
]
p = Popen(args, stdin=stdin, stdout=stdout)
if p.wait() != 0:
return None
stdout.seek(0)
graph = stdout.read()
return graph_from_dot_data(graph)
def run_command(self, command, stdin=None, env=None):
"""
Launch a shell command line.
:param command: Command line to launch
:type command: str
:param stdin: Standard input of command
:type stdin: file
:param env: Environment variable used in command
:type env: dict
:return: Standard output of command
:rtype: file
"""
cmd = shlex.split(command)
stdout = SpooledTemporaryFile(max_size=settings.TMP_FILE_MAX_SIZE,
dir=settings.TMP_DIR)
stderr = SpooledTemporaryFile(max_size=settings.TMP_FILE_MAX_SIZE,
dir=settings.TMP_DIR)
full_env = self.env.copy()
full_env.update(env or {})
try:
process = Popen(cmd, stdin=stdin, stdout=stdout, stderr=stderr,
env=full_env)
process.wait()
if process.poll():
stderr.seek(0)
raise exceptions.CommandConnectorError(
"Error running: {}\n{}".format(command, stderr.read()))
stdout.seek(0)
stderr.seek(0)
return stdout, stderr
except OSError as err:
raise exceptions.CommandConnectorError(
"Error running: {}\n{}".format(command, str(err)))
class InputStream(object):
"""
FCGI_STDIN or FCGI_DATA stream.
Uses temporary file to store received data once max_mem bytes
have been received.
"""
def __init__(self, max_mem=1024):
self._file = SpooledTemporaryFile(max_mem)
self._eof_received = Event()
def feed(self, data):
if self._eof_received.is_set():
raise IOError('Feeding file beyond EOF mark')
if not data: # EOF mark
self._file.seek(0)
self._eof_received.set()
else:
self._file.write(data)
def __iter__(self):
self._eof_received.wait()
return iter(self._file)
def read(self, size=-1):
self._eof_received.wait()
return self._file.read(size)
def readlines(self, sizehint=0):
self._eof_received.wait()
return self._file.readlines(sizehint)
@property
def eof_received(self):
return self._eof_received.is_set()
def _shell_command(self, cmd): """Shell out a subprocess and return what it writes to stdout as a string""" in_mem_file = SpooledTemporaryFile(max_size=2048, mode="r+") check_call(cmd, shell=True, stdout=in_mem_file) in_mem_file.seek(0) stdout = in_mem_file.read() in_mem_file.close() return stdout
def decode_bytes(bytes):
temp = SpooledTemporaryFile()
string = ""
for byte in bytes:
string = string + chr(byte)
temp.write(string)
temp.seek(0) # work around a stupid python bug
return Decode(0, temp.read(), Decode64Bits)
def image(type, spec=' ', ext='png'):
# Parameters for `suml`.
import suml.common
import optparse
options = optparse.Values(({
'scruffy': True,
'png': ext == 'png',
'svg': ext == 'svg' or ext == 'pdf',
'font': os.getenv('SCRUFFY_FONT', suml.common.defaultScruffyFont()),
'shadow': False,
}))
from tempfile import SpooledTemporaryFile
fout = SpooledTemporaryFile()
# Execute Scruffy `suml`.
if type == 'class':
suml.yuml2dot.transform(spec, fout, options)
elif type == 'sequence':
suml.suml2pic.transform(spec, fout, options)
else:
return HTTPError(404, 'Unhandled diagram type.')
# Retrieve the data generated.
fout.seek(0)
data = fout.read()
fout.close()
# Convert SVG to PDF?
if ext == 'pdf':
# Load SVG file.
doc = xml.dom.expatbuilder.parseString(data)
# Convert to a RLG drawing
svg_renderer = svglib.svglib.SvgRenderer()
svg_renderer.render(doc.documentElement)
drawing = svg_renderer.finish()
# Generate PDF.
data = reportlab.graphics.renderPDF.drawToString(drawing)
# Server the generated image.
if ext == 'png':
response.content_type = 'image/png'
elif ext == 'svg':
response.content_type = 'image/svg+xml'
elif ext == 'pdf':
response.content_type = 'application/pdf'
else:
return HTTPError(500, 'Unhandled extension type.')
return data
class StartFinish(object):
def __init__(self):
pass
def start(self, args):
self.outFile = SpooledTemporaryFile()
self.errFile = SpooledTemporaryFile()
self.cmdline = list2cmdline(args)
print 'starting: ' + self.cmdline
self.process = Popen(args,
stderr=self.errFile, stdout=self.outFile, universal_newlines=False)
self.process_start = time()
def finish(self, timeout):
print 'finishing "' + self.cmdline + '"'
kill_time = self.process_start + timeout
self.process_end = time()
while self.process.poll() is None:
self.process_end = time()
if self.process_end < kill_time:
sleep(0.1)
else:
self.process.kill()
raise Exception('timeout "' + self.cmdline + '"')
# read temp streams from start
self.outFile.seek(0)
self.errFile.seek(0)
self.out = self.outFile.read()
self.err = self.errFile.read()
self.outFile.close()
self.errFile.close()
print 'out:', self.out
print 'err:', self.err
print 'returncode:', self.process.returncode
print 'elapsed:', self.process_end - self.process_start
return self.process.returncode
def csv_fn(pydata):
csvfile=SpooledTemporaryFile()
csvw=csv.writer(csvfile)
fieldnames=[]
for h in pydata:
for k in pydata[h]:
if k not in fieldnames:
fieldnames.append(k)
if showheader:
csvw.writerow(['system_name'] + fieldnames)
for system_name in pydata:
csvw.writerow([system_name] + [pydata[system_name].get(k, None) for k in fieldnames])
csvfile.seek(0)
results=csvfile.read()
csvfile.close()
return results
def run(args, timeout = 10):
'''
Run a command with a timeout after which it will be forcibly
killed.
'''
out_file = SpooledTemporaryFile()
p = Popen(args, shell=True, stdout=out_file, stderr=STDOUT)
wait_remaining_sec = timeout
while p.poll() is None and wait_remaining_sec > 0:
sleep(1)
wait_remaining_sec -= 1
if wait_remaining_sec <= 0:
kill_proc(p.pid)
r = -9
else:
r = p.returncode
out_file.seek(0)
out = out_file.read()
return out, r
class LLDB:
def _parseStackTrace(self, jibberish):
not_jibberish = re.findall(r'\(lldb\) bt(.*)\(lldb\)', jibberish, re.DOTALL)
if len(not_jibberish) != 0:
return not_jibberish[0].replace(' frame ', '')
else:
return 'Stack Trace failed:', jibberish
def _waitForResponse(self, wait=True):
while wait:
self.lldb_stdout.seek(self.last_position)
for line in self.lldb_stdout:
if line == '(lldb) ':
self.last_position = self.lldb_stdout.tell()
return True
time.sleep(0.05)
time.sleep(0.05)
return True
def getStackTrace(self, pid):
lldb_commands = [ 'attach -p ' + pid + '\n', 'bt\n', 'quit\n', 'Y\n' ]
self.lldb_stdout = SpooledTemporaryFile()
self.last_position = 0
lldb_process = subprocess.Popen(['lldb', '-x'], stdin=subprocess.PIPE, stdout=self.lldb_stdout, stderr=self.lldb_stdout)
while lldb_process.poll() == None:
for command in lldb_commands:
if command == lldb_commands[-1]:
lldb_commands = []
if self._waitForResponse(False):
# I have seen LLDB exit out from under us
try:
lldb_process.stdin.write(command)
except:
pass
elif self._waitForResponse():
lldb_process.stdin.write(command)
self.lldb_stdout.seek(0)
stack_trace = self._parseStackTrace(self.lldb_stdout.read())
self.lldb_stdout.close()
return stack_trace
class GDB:
def _parseStackTrace(self, jibberish):
not_jibberish = re.findall(r'\(gdb\) (#.*)\(gdb\)', jibberish, re.DOTALL)
if len(not_jibberish) != 0:
return not_jibberish[0]
else:
return 'Stack Trace failed:', jibberish
def _waitForResponse(self, wait=True):
while wait:
self.gdb_stdout.seek(self.last_position)
for line in self.gdb_stdout:
if line == '(gdb) ':
self.last_position = self.gdb_stdout.tell()
return True
time.sleep(0.05)
time.sleep(0.05)
return True
def getStackTrace(self, pid):
gdb_commands = [ 'attach ' + pid + '\n', 'set verbose off\n', 'thread\n', 'apply\n', 'all\n', 'bt\n', 'quit\n', 'y\n' ]
self.gdb_stdout = SpooledTemporaryFile()
self.last_position = 0
gdb_process = subprocess.Popen(['gdb', '-nx'], stdin=subprocess.PIPE, stdout=self.gdb_stdout, stderr=self.gdb_stdout)
while gdb_process.poll() == None:
for command in gdb_commands:
if command == gdb_commands[-1]:
gdb_commands = []
elif self._waitForResponse():
# I have seen GDB exit out from under us
try:
gdb_process.stdin.write(command)
except:
pass
self.gdb_stdout.seek(0)
stack_trace = self._parseStackTrace(self.gdb_stdout.read())
self.gdb_stdout.close()
return stack_trace
def load_stix(stix):
# Just save the pain and load it if the first character is a <
log.debug("Loading STIX...")
if sys.version_info < (3, 5):
json_exception = ValueError
else:
json_exception = json.JSONDecodeError
if isinstance(stix, STIXPackage):
log.debug("Argument was already STIX package, ignoring.")
# Oh cool we're ok
# Who tried to load this? Honestly.
return stix
elif hasattr(stix, 'read'):
log.debug("Argument has 'read' attribute, assuming file-like.")
# It's a file!
# But somehow, sometimes, reading it returns a bytes stream
# and the loader dies on python 3.4.
# Luckily, STIXPackage.from_json (which is mixbox.Entity.from_json)
# will happily load a string.
# So we're going to play dirty.
data = stix.read()
log.debug("Read file, type %s.", type(data))
if isinstance(data, bytes):
data = data.decode()
try:
log.debug("Attempting to load from JSON...")
# Try loading from JSON
stix_package = STIXPackage.from_json(data)
except json_exception:
log.debug("Attempting to load from XML...")
# Ok then try loading from XML
# Loop zoop
# Read the STIX into an Etree
stix.seek(0)
stix_xml = etree.fromstring(stix.read())
ns_map = stix_xml.nsmap
# Remove any "marking" sections because the US-Cert is evil
log.debug("Removing Marking elements...")
pattern = ".//{http://data-marking.mitre.org/Marking-1}Marking"
for element in stix_xml.findall(pattern):
element.getparent().remove(element)
log.debug("Writing cleaned XML to Tempfile")
f = SpooledTemporaryFile(max_size=10 * 1024)
f.write(etree.tostring(stix_xml))
f.seek(0)
# Pray to anything you hold sacred
ns_objmap = map(lambda x: Namespace(ns_map[x], x), ns_map)
for ns in ns_objmap:
log.debug("Trying to add namespace %s", ns)
try:
nsparser.STIX_NAMESPACES.add_namespace(ns)
mixbox.namespaces.register_namespace(ns)
except Exception as ex:
log.exception(ex)
try:
log.debug("Attempting to read clean XML into STIX...")
stix_package = STIXPackage.from_xml(f)
except Exception as ex:
# No joy. Quit.
log.fatal("Could not :<")
log.exception(ex)
f.seek(0)
with open("FAILED_STIX.xml", "wb") as g:
g.write(f.read())
raise STIXLoadError("Could not load stix file. {}".format(ex))
return stix_package
elif isinstance(stix, (str, bytes)):
if isinstance(stix, bytes):
stix = stix.decode()
# It's text, we'll need to use a temporary file
# Create a temporary file to load from
# Y'know I should probably give it a max size before jumping to disk
# idk, 10MB? Sounds reasonable.
f = SpooledTemporaryFile(max_size=10 * 1024)
# O I have idea for sneak
# Will be very sneak
# Write the (probably) XML to file
f.write(stix.encode("utf-8"))
# Reset the file so we can read from it
f.seek(0)
# AHA SNEAK DIDN'T EXPECT RECURSION DID YOU
return load_stix(f)
class Buffer(FileWrapper):
"""Class implementing buffering of input and output streams.
This class uses a separate buffer file to hold the contents of the
underlying file while they are being manipulated. As data is read
it is duplicated into the buffer, and data is written from the buffer
back to the file on close.
"""
def __init__(self, fileobj, mode=None, max_size_in_memory=1024 * 8):
"""Buffered file wrapper constructor."""
self._buffer = SpooledTemporaryFile(max_size=max_size_in_memory)
self._in_eof = False
self._in_pos = 0
self._was_truncated = False
super(Buffer, self).__init__(fileobj, mode)
def _buffer_size(self):
try:
return len(self._buffer.file.getvalue())
except AttributeError:
return os.fstat(self._buffer.fileno()).st_size
def _buffer_chunks(self):
chunk = self._buffer.read(16 * 1024)
if chunk == "":
yield chunk
else:
while chunk != "":
yield chunk
chunk = self._buffer.read(16 * 1024)
def _write_out_buffer(self):
if self._check_mode("r"):
self._read_rest()
if "a" in self.mode:
self._buffer.seek(self._in_pos)
self._fileobj.seek(self._in_pos)
else:
self._fileobj.seek(0)
self._buffer.seek(0)
else:
self._buffer.seek(0)
if self._was_truncated:
self._fileobj.truncate(0)
self._was_truncated = False
for chunk in self._buffer_chunks():
self._fileobj.write(chunk)
def flush(self):
# flush the buffer; we only write to the underlying file on close
self._buffer.flush()
def close(self):
if self.closed:
return
if self._check_mode("w"):
self._write_out_buffer()
super(Buffer, self).close()
self._buffer.close()
def _read(self, sizehint=-1):
# First return any data available from the buffer.
# Since we don't flush the buffer after every write, certain OSes
# (guess which!) will happily read junk data from the end of it.
# Instead, we explicitly read only up to self._in_pos.
if not self._in_eof:
buffered_size = self._in_pos - self._buffer.tell()
if sizehint >= 0:
buffered_size = min(sizehint, buffered_size)
else:
buffered_size = sizehint
data = self._buffer.read(buffered_size)
if data != "":
return data
# Then look for more data in the underlying file
if self._in_eof:
return None
data = self._fileobj.read(sizehint)
self._in_pos += len(data)
self._buffer.write(data)
if sizehint < 0 or len(data) < sizehint:
self._in_eof = True
self._buffer.flush()
return data
def _write(self, data, flushing=False):
self._buffer.write(data)
if self._check_mode("r") and not self._in_eof:
diff = self._buffer.tell() - self._in_pos
if diff > 0:
junk = self._fileobj.read(diff)
self._in_pos += len(junk)
if len(junk) < diff:
self._in_eof = True
self._buffer.flush()
def _seek(self, offset, whence):
# Ensure we've read enough to simply do the seek on the buffer
if self._check_mode("r") and not self._in_eof:
if whence == 0:
if offset > self._in_pos:
self._read_rest()
if whence == 1:
if self._buffer.tell() + offset > self._in_pos:
self._read_rest()
if whence == 2:
self._read_rest()
# Then just do it on the buffer...
self._buffer.seek(offset, whence)
def _tell(self):
return self._buffer.tell()
def _truncate(self, size):
if self._check_mode("r") and not self._in_eof:
if size > self._in_pos:
self._read_rest()
self._in_eof = True
try:
self._buffer.truncate(size)
except TypeError:
et, ev, tb = sys.exc_info()
# SpooledTemporaryFile.truncate() doesn't accept size paramter.
try:
self._buffer._file.truncate(size)
except Exception:
raise et, ev, tb
# StringIO objects don't truncate to larger size correctly.
if hasattr(self._buffer, "_file"):
_file = self._buffer._file
if hasattr(_file, "getvalue"):
if len(_file.getvalue()) != size:
curpos = _file.tell()
_file.seek(0, 2)
_file.write("\x00" * (size - len(_file.getvalue())))
_file.seek(curpos)
self._was_truncated = True
def _read_rest(self):
"""Read the rest of the input stream."""
if self._in_eof:
return
pos = self._buffer.tell()
self._buffer.seek(0, 2)
data = self._fileobj.read(self._bufsize)
while data:
self._in_pos += len(data)
self._buffer.write(data)
data = self._fileobj.read(self._bufsize)
self._in_eof = True
self._buffer.flush()
self._buffer.seek(pos)
class UploadFile:
"""
An uploaded file included as part of the request data.
"""
__slots__ = ("filename", "content_type", "file")
spool_max_size = 1024 * 1024
def __init__(self, filename: str, content_type: str = "") -> None:
self.filename = filename
self.content_type = content_type
self.file = SpooledTemporaryFile(max_size=self.spool_max_size,
mode="w+b")
@property
def in_memory(self) -> bool:
rolled_to_disk = getattr(self.file, "_rolled", True)
return not rolled_to_disk
def write(self, data: bytes) -> None:
self.file.write(data)
async def awrite(self, data: bytes) -> None:
if self.in_memory:
self.write(data)
else:
await asyncio.get_event_loop().run_in_executor(
None, self.write, data)
def read(self, size: int = -1) -> bytes:
return self.file.read(size)
async def aread(self, size: int = -1) -> bytes:
if self.in_memory:
return self.read(size)
return await asyncio.get_event_loop().run_in_executor(
None, self.read, size)
def seek(self, offset: int) -> None:
self.file.seek(offset)
async def aseek(self, offset: int) -> None:
if self.in_memory:
self.seek(offset)
else:
await asyncio.get_event_loop().run_in_executor(
None, self.seek, offset)
def close(self) -> None:
self.file.close()
async def aclose(self) -> None:
if self.in_memory:
self.close()
else:
await asyncio.get_event_loop().run_in_executor(None, self.close)
def save(self, filepath: str) -> None:
"""
Save file to disk.
"""
# from shutil.COPY_BUFSIZE
copy_bufsize = 1024 * 1024 if os.name == "nt" else 64 * 1024
file_position = self.file.tell()
self.file.seek(0, 0)
try:
with open(filepath, "wb+") as target_file:
source_read = self.file.read
target_write = target_file.write
while True:
buf = source_read(copy_bufsize)
if not buf:
break
target_write(buf)
finally:
self.file.seek(file_position)
async def asave(self, filepath: str) -> None:
"""
Save file to disk, work in threading pool.
"""
await asyncio.get_event_loop().run_in_executor(None, self.save,
filepath)
def do_execute(self,
code,
silent,
store_history=True,
user_expressions=None,
allow_stdin=False):
"""Execute user code."""
if len(code.strip()) == 0:
return {
'status': 'ok',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {}
}
env = builtins.__xonsh_env__
shell = builtins.__xonsh_shell__
hist = builtins.__xonsh_history__
enc = env.get('XONSH_ENCODING')
out = SpooledTemporaryFile(max_size=MAX_SIZE,
mode='w+t',
encoding=enc,
newline='\n')
err = SpooledTemporaryFile(max_size=MAX_SIZE,
mode='w+t',
encoding=enc,
newline='\n')
try:
with redirect_stdout(out), redirect_stderr(err), \
swap(builtins, '__xonsh_stdout_uncaptured__', out), \
swap(builtins, '__xonsh_stderr_uncaptured__', err), \
env.swap({'XONSH_STORE_STDOUT': False}):
shell.default(code)
interrupted = False
except KeyboardInterrupt:
interrupted = True
if not silent: # stdout response
if out.tell() > 0:
out.seek(0)
self._respond_in_chunks('stdout', out.read())
if err.tell() > 0:
err.seek(0)
self._respond_in_chunks('stderr', err.read())
if hasattr(builtins, '_') and builtins._ is not None:
# rely on sys.displayhook functionality
self._respond_in_chunks('stdout', pformat(builtins._))
builtins._ = None
if len(hist) > 0 and out.tell() == 0 and err.tell() == 0:
self._respond_in_chunks('stdout', hist.outs[-1])
out.close()
err.close()
if interrupted:
return {'status': 'abort', 'execution_count': self.execution_count}
rtn = 0 if len(hist) == 0 else hist.rtns[-1]
if 0 < rtn:
message = {
'status': 'error',
'execution_count': self.execution_count,
'ename': '',
'evalue': str(rtn),
'traceback': []
}
else:
message = {
'status': 'ok',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {}
}
return message
def process(self, host, s):
encrypted_options = ''
while True:
data = s.recv(8192)
if data == b'END_EVENT':
break
else:
encrypted_options = encrypted_options + data
try:
options = self.decrypt_options(encrypted_options)
except:
s.close()
return
soc = socket(AF_INET, SOCK_STREAM)
soc.setblocking(0)
soc.settimeout(int(self.config['Event_Receiver']['PCAP']['timeout']))
db = self.core.get_db()
sensors = db.sensors
client_info = sensors.find_one( { "SERVER": options['sensor'] })
client_cert = client_info['cert']
cert_tmp = NamedTemporaryFile(mode='w+b', suffix='.pem')
cert_tmp.write(client_cert)
cert_tmp.flush()
soc_ssl = ssl.wrap_socket(soc, ca_certs=cert_tmp.name, cert_reqs=ssl.CERT_REQUIRED, ssl_version=ssl.PROTOCOL_SSLv3)
encrypted_options = self.encrypt_requests(self.config, options)
try:
soc_ssl.connect((client_info['IP'], int(client_info['sensor_port'])))
except error:
s.send('Sensor cannot be reached')
s.close()
return
soc_ssl.send(encrypted_options)
soc_ssl.send('END_EVENT')
tmp_file = SpooledTemporaryFile(mode='wb')
while True:
try:
data = soc_ssl.recv(8192)
except ssl.SSLError:
s.send('Request Timed Out')
s.close()
return
if data == b'END_EVENT':
break
elif data != 'No Packets Found':
tmp_file.write(data)
else:
s.send('No Packets Found')
s.close()
cert_tmp.close()
return
tmp_file.seek(0)
pcap = tmp_file.read(8192)
try:
while (pcap):
s.send(pcap)
pcap = tmp_file.read(8192)
s.send(b'END_EVENT')
except error:
s.close()
return
s.close()
cert_tmp.close()
return
def binary_benchmark(self, function):
"""
:param function:
:type function:
"""
getattr(self, function)()
# Strip
stripper = os.environ[self.compiler_bin].replace("gcc", "strip")
flags = []
flags.append(self.filename[1])
cmd = [stripper] + flags
tfile = SpooledTemporaryFile()
error_code = subprocess.check_call(cmd,
stdout=tfile,
stderr=subprocess.STDOUT)
if error_code != 0:
tfile.seek(0)
print(tfile.read())
self.assertEqual(error_code, 0)
# Disassemble
disassembler = os.environ[self.compiler_bin].replace("gcc", "objdump")
if self.dump_flags in os.environ:
flags = os.environ[self.dump_flags].split(" ")
else:
flags = []
flags.append("-w")
flags.append("-d")
flags.append(self.filename[1])
cmd = [disassembler] + flags
output = subprocess.check_output(cmd)
if six.PY3:
output = output.decode()
output_lines = output.split("\n")
asmline = ""
asmline_bin = ""
for idx, line in enumerate(output_lines[6:]):
if line.strip() == "":
continue
line = line.split(":")[1].strip()
line_bin = line.split("\t")[0]
if idx % 2 == 1:
asmline = line
asmline_bin = line_bin
else:
print(self.filename[1])
print("'%s' <-> '%s'" % (line, asmline))
print("'%s' = '%s' ?" % (line_bin, asmline_bin))
self.assertEqual(line_bin, asmline_bin)
def compile_benchmark(self, function):
"""
:param function:
:type function:
"""
getattr(self, function)()
# Compile
compiler = os.environ[self.compiler_bin]
if self.compiler_flags in os.environ:
flags = os.environ[self.compiler_flags].split(" ")
else:
flags = []
flags.append("-c")
flags.append(self.filename[0])
flags.append("-o")
flags.append(self.filename[0].replace(".s", ".o"))
cmd = [compiler] + flags
tfile = SpooledTemporaryFile()
try:
error_code = subprocess.check_call(cmd,
stdout=tfile,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as exc:
error_code = exc.returncode
if error_code == 0:
self.filename.append(self.filename[0].replace(".s", ".o"))
return
else:
tfile.seek(0)
print("Compiler output:")
print(tfile.read(0))
# Assemble directly (it might be needed for some situations, where
# the gcc is not updated but GNU gas is updated
# Assemble
assembler = os.environ[self.compiler_bin].replace("gcc", "as")
if self.asm_flags in os.environ:
flags = os.environ[self.asm_flags].split(" ")
else:
flags = []
flags.append(self.filename[0])
flags.append("-o")
flags.append(self.filename[0].replace(".s", ".o"))
cmd = [assembler] + flags
error_code = 0
try:
cmd_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as exc:
error_code = exc.returncode
cmd_output = exc.output
if six.PY3:
cmd_output = cmd_output.decode()
if error_code == 0:
self.filename.append(self.filename[0].replace(".s", ".o"))
return
# Analyze the output to check if this fail is related to microprobe
# or related to the tool-chain used
if cmd_output.find("Error: unrecognized opcode:") > -1:
# Compiler not new enough or check compilation options
# DO NOT REPORT FAILURE but not PASS
self.fail(msg="Update your toolchain: %s not supported\n%s" %
(self.instr_name, _process_as_output(cmd_output)))
self.fail(msg="Error compiling using cmd: %s. Output: %s" %
(cmd, _process_as_output(cmd_output)))
class IncrementalWriter:
"""A streaming file writer for point clouds.
Using the IncrementalWriter with spooled temporary files, which are
only flushed to disk if they go above the given size, allows for
streaming points to disk even when the header is unknown in advance.
This allows some nice tricks, including splitting a point cloud into
multiple files in a single pass, without memory issues.
"""
# pylint:disable=too-few-public-methods
def __init__(self,
filename: str,
header: PlyHeader,
utm: UTM_Coord = None,
buffer=2**22) -> None:
"""
Args:
filename: final place to save the file on disk.
source_fname: source file for the pointcloud; used to detect
file format for metadata etc.
buffer (int): The number of bytes to hold in RAM before flushing
the temporary file to disk. Default 1MB, which holds ~8300
points - enough for most objects but still practical to hold
thousands in memory. Set a smaller buffer for large forests.
"""
self.filename = filename
self.temp_storage = SpooledTemporaryFile(max_size=buffer, mode='w+b')
self.count = 0
self.utm = utm
self.header = header
# Always write in big-endian mode; only store type information
self.binary = struct.Struct('>' + header.form_str[1:])
def __call__(self, point) -> None:
"""Add a single point to this pointcloud, saving in binary format.
Args:
point (namedtuple): vertex attributes for the point, eg xyzrgba.
"""
self.temp_storage.write(self.binary.pack(*point))
self.count += 1
def __del__(self):
"""Flush data to disk and clean up."""
to_ply_types = {v: k for k, v in PLY_TYPES.items()}
properties = [
'property {t} {n}'.format(t=t, n=n) for t, n in zip((
to_ply_types[p]
for p in self.header.form_str[1:]), self.header.names)
]
head = [
'ply', 'format binary_big_endian 1.0',
'element vertex {}'.format(self.count), '\n'.join(properties),
'end_header'
]
if self.utm is not None:
head.insert(
-1, 'comment UTM x y zone north ' +
'{0.x} {0.y} {0.zone} {0.north}'.format(self.utm))
if not os.path.isdir(os.path.dirname(self.filename)):
os.makedirs(os.path.dirname(self.filename))
with open(self.filename, 'wb') as f:
f.write(('\n'.join(head) + '\n').encode('ascii'))
self.temp_storage.seek(0)
chunk = self.temp_storage.read(8192)
while chunk:
f.write(chunk)
chunk = self.temp_storage.read(8192)
self.temp_storage.close()
class TestFile:
CACHE_LIMIT = 0x80000 # data cache limit per file: 512KB
XFER_BUF = 0x10000 # transfer buffer size: 64KB
__slots__ = ("_file_name", "_fp")
def __init__(self, file_name):
# This is a naive fix for a larger path issue. This is a simple sanity
# check and does not check if invalid characters are used. If an invalid
# file name is used an exception will be raised when trying to write
# that file to the file system.
if "\\" in file_name:
file_name = file_name.replace("\\", "/")
if file_name.startswith("/"):
file_name = file_name.lstrip("/")
if file_name.endswith("."):
file_name = file_name.rstrip(".")
if not file_name \
or ("/" in file_name and not file_name.rsplit("/", 1)[-1]) \
or file_name.startswith("../"):
raise TypeError("file_name is invalid %r" % (file_name, ))
# name including path relative to wwwroot
self._file_name = normpath(file_name)
self._fp = SpooledTemporaryFile(dir=grz_tmp("storage"),
max_size=self.CACHE_LIMIT,
prefix="testfile_")
def __enter__(self):
return self
def __exit__(self, *exc):
self.close()
def clone(self):
"""Make a copy of the TestFile.
Args:
None
Returns:
TestFile: A copy of the TestFile instance
"""
cloned = type(self)(self._file_name)
self._fp.seek(0)
copyfileobj(self._fp, cloned._fp, self.XFER_BUF) # pylint: disable=protected-access
return cloned
def close(self):
"""Close the TestFile.
Args:
None
Returns:
None
"""
self._fp.close()
@property
def data(self):
"""Get the data from the TestFile. Not recommenced for large files.
Args:
None
Returns:
bytes: Data from the TestFile
"""
pos = self._fp.tell()
self._fp.seek(0)
data = self._fp.read()
self._fp.seek(pos)
return data
def dump(self, path):
"""Write TestFile data to the filesystem.
Args:
path (str): Path to output data.
Returns:
None
"""
target_path = pathjoin(path, dirname(self._file_name))
if not isdir(target_path):
makedirs(target_path)
self._fp.seek(0)
with open(pathjoin(path, self._file_name), "wb") as dst_fp:
copyfileobj(self._fp, dst_fp, self.XFER_BUF)
@property
def file_name(self):
return self._file_name
@classmethod
def from_data(cls, data, file_name, encoding="UTF-8"):
"""Create a TestFile and add it to the test case.
Args:
data (bytes or str): Data to write to file. If data is of type str
encoding must be given.
file_name (str): Name for the TestFile.
encoding (str): Encoding to be used.
Returns:
TestFile: A TestFile.
"""
t_file = cls(file_name)
if data:
if isinstance(data, bytes) or not encoding:
t_file.write(data)
else:
t_file.write(data.encode(encoding))
return t_file
@classmethod
def from_file(cls, input_file, file_name=None):
"""Create a TestFile from an existing file.
Args:
input_file (str): Path to existing file to use.
file_name (str): Name for the TestFile. If file_name is not given
the name of the input_file will be used.
Returns:
TestFile: A TestFile.
"""
if file_name is None:
file_name = basename(input_file)
t_file = cls(file_name)
with open(input_file, "rb") as src_fp:
copyfileobj(src_fp, t_file._fp, cls.XFER_BUF) # pylint: disable=protected-access
return t_file
@property
def size(self):
"""Size of the file in bytes.
Args:
None
Returns:
int: Size in bytes.
"""
pos = self._fp.tell()
self._fp.seek(0, SEEK_END)
size = self._fp.tell()
self._fp.seek(pos)
return size
def write(self, data):
"""Add data to the TestFile.
Args:
data (bytes): Data to add to the TestFile.
Returns:
None
"""
self._fp.write(data)
class TPCTemporaryStorage(object):
__slots__ = (
'_queue',
'_queue_contents',
)
def __init__(self):
# start with a fresh in-memory buffer instead of reusing one that might
# already be spooled to disk.
# TODO: An alternate idea would be a temporary sqlite database.
self._queue = SpooledTemporaryFile(max_size=10 * 1024 * 1024)
# {oid: (startpos, endpos, prev_tid_int)}
self._queue_contents = OidObjectMap()
def reset(self):
self._queue_contents.clear()
self._queue.seek(0)
def store_temp(self, oid_int, state, prev_tid_int=0):
"""
Queue an object for caching.
Typically, we can't actually cache the object yet, because its
transaction ID is not yet chosen.
"""
queue = self._queue
queue.seek(0, 2) # seek to end
startpos = queue.tell()
queue.write(state)
endpos = queue.tell()
self._queue_contents[oid_int] = (startpos, endpos, prev_tid_int)
def __len__(self):
# How many distinct OIDs have been stored?
# This also lets us be used in a boolean context to see
# if we've actually stored anything or are closed.
return len(self._queue_contents)
@property
def stored_oids(self):
return self._queue_contents
@property
def max_stored_oid(self):
return OidObjectMap_max_key(self._queue_contents)
def _read_temp_state(self, startpos, endpos):
self._queue.seek(startpos)
length = endpos - startpos
state = self._queue.read(length)
if len(state) != length:
raise AssertionError("Queued cache data is truncated")
return state
def read_temp(self, oid_int):
"""
Return the bytes for a previously stored temporary item.
"""
startpos, endpos, _ = self._queue_contents[oid_int]
return self._read_temp_state(startpos, endpos)
def __iter__(self):
return self.iter_for_oids(None)
def iter_for_oids(self, oids):
read_temp_state = self._read_temp_state
for startpos, endpos, oid_int, prev_tid_int in self.items(oids):
state = read_temp_state(startpos, endpos)
yield state, oid_int, prev_tid_int
def items(self, oids=None):
# Order the queue by file position, which should help
# if the file is large and needs to be read
# sequentially from disk.
items = [(startpos, endpos, oid_int, prev_tid_int)
for (oid_int,
(startpos, endpos,
prev_tid_int)) in iteroiditems(self._queue_contents)
if oids is None or oid_int in oids]
items.sort()
return items
def close(self):
if self._queue is not None:
self._queue.close()
self._queue = None
self._queue_contents = () # Not None so len() keeps working
def __repr__(self):
approx_size = 0
if self._queue is not None:
self._queue.seek(0, 2) # seek to end
# The number of bytes we stored isn't necessarily the
# number of bytes we send to the server, if there are duplicates
approx_size = self._queue.tell()
return "<%s at 0x%x count=%d bytes=%d>" % (
type(self).__name__, id(self), len(self), approx_size)
def __str__(self):
base = repr(self)
if not self:
return base
out = NStringIO()
div = '=' * len(base)
headings = ['OID', 'Length', 'Previous TID']
col_width = (len(base) - 5) // len(headings)
print(base, file=out)
print(div, file=out)
print('| ', file=out, end='')
for heading in headings:
print('%-*s' % (col_width, heading), end='', file=out)
print('| ', end='', file=out)
out.seek(out.tell() - 3)
print('|', file=out)
print(div, file=out)
items = sorted(
(oid_int, endpos - startpos, prev_tid_int)
for (startpos, endpos, oid_int, prev_tid_int) in self.items())
for oid_int, length, prev_tid_int in items:
print('%*d |%*d |%*d' % (col_width, oid_int, col_width, length,
col_width, prev_tid_int),
file=out)
return out.getvalue()
class IncrementalWriter:
"""A streaming file writer for point clouds.
Using the IncrementalWriter with spooled temporary files, which are
only flushed to disk if they go above the given size, allows for
streaming points to disk even when the header is unknown in advance.
This allows some nice tricks, including splitting a point cloud into
multiple files in a single pass, without memory issues.
"""
# pylint:disable=too-few-public-methods
def __init__(self, filename: str, header: PlyHeader,
utm: UTM_Coord=None, buffer=2**22) -> None:
"""
Args:
filename: final place to save the file on disk.
source_fname: source file for the pointcloud; used to detect
file format for metadata etc.
buffer (int): The number of bytes to hold in RAM before flushing
the temporary file to disk. Default 1MB, which holds ~8300
points - enough for most objects but still practical to hold
thousands in memory. Set a smaller buffer for large forests.
"""
self.filename = filename
self.temp_storage = SpooledTemporaryFile(max_size=buffer, mode='w+b')
self.count = 0
self.utm = utm
self.header = header
# Always write in big-endian mode; only store type information
self.binary = struct.Struct('>' + header.form_str[1:])
def __call__(self, point) -> None:
"""Add a single point to this pointcloud, saving in binary format.
Args:
point (namedtuple): vertex attributes for the point, eg xyzrgba.
"""
self.temp_storage.write(self.binary.pack(*point))
self.count += 1
def __del__(self):
"""Flush data to disk and clean up."""
to_ply_types = {v: k for k, v in PLY_TYPES.items()}
properties = ['property {t} {n}'.format(t=t, n=n) for t, n in zip(
(to_ply_types[p] for p in self.header.form_str[1:]),
self.header.names)]
head = ['ply',
'format binary_big_endian 1.0',
'element vertex {}'.format(self.count),
'\n'.join(properties),
'end_header']
if self.utm is not None:
head.insert(-1, 'comment UTM x y zone north ' +
'{0.x} {0.y} {0.zone} {0.north}'.format(self.utm))
if not os.path.isdir(os.path.dirname(self.filename)):
os.makedirs(os.path.dirname(self.filename))
with open(self.filename, 'wb') as f:
f.write(('\n'.join(head) + '\n').encode('ascii'))
self.temp_storage.seek(0)
chunk = self.temp_storage.read(8192)
while chunk:
f.write(chunk)
chunk = self.temp_storage.read(8192)
self.temp_storage.close()
class Buffer(FileWrapper):
"""Class implementing buffering of input and output streams.
This class uses a separate buffer file to hold the contents of the
underlying file while they are being manipulated. As data is read
it is duplicated into the buffer, and data is written from the buffer
back to the file on close.
"""
def __init__(self, fileobj, mode=None, max_size_in_memory=1024 * 8):
"""Buffered file wrapper constructor."""
self._buffer = SpooledTemporaryFile(max_size=max_size_in_memory)
self._in_eof = False
self._in_pos = 0
self._was_truncated = False
super(Buffer, self).__init__(fileobj, mode)
def _buffer_size(self):
try:
return len(self._buffer.file.getvalue())
except AttributeError:
return os.fstat(self._buffer.fileno()).st_size
def _buffer_chunks(self):
chunk = self._buffer.read(16 * 1024)
if chunk == "":
yield chunk
else:
while chunk != "":
yield chunk
chunk = self._buffer.read(16 * 1024)
def _write_out_buffer(self):
if self._check_mode("r"):
self._read_rest()
if "a" in self.mode:
self._buffer.seek(self._in_pos)
self._fileobj.seek(self._in_pos)
else:
self._fileobj.seek(0)
self._buffer.seek(0)
else:
self._buffer.seek(0)
if self._was_truncated:
self._fileobj.truncate(0)
self._was_truncated = False
for chunk in self._buffer_chunks():
self._fileobj.write(chunk)
def flush(self):
# flush the buffer; we only write to the underlying file on close
self._buffer.flush()
def close(self):
if self.closed:
return
if self._check_mode("w"):
self._write_out_buffer()
super(Buffer, self).close()
self._buffer.close()
def _read(self, sizehint=-1):
# First return any data available from the buffer.
# Since we don't flush the buffer after every write, certain OSes
# (guess which!) will happily read junk data from the end of it.
# Instead, we explicitly read only up to self._in_pos.
if not self._in_eof:
buffered_size = self._in_pos - self._buffer.tell()
if sizehint >= 0:
buffered_size = min(sizehint, buffered_size)
else:
buffered_size = sizehint
data = self._buffer.read(buffered_size)
if data != "":
return data
# Then look for more data in the underlying file
if self._in_eof:
return None
data = self._fileobj.read(sizehint)
self._in_pos += len(data)
self._buffer.write(data)
if sizehint < 0 or len(data) < sizehint:
self._in_eof = True
self._buffer.flush()
return data
def _write(self, data, flushing=False):
self._buffer.write(data)
if self._check_mode("r") and not self._in_eof:
diff = self._buffer.tell() - self._in_pos
if diff > 0:
junk = self._fileobj.read(diff)
self._in_pos += len(junk)
if len(junk) < diff:
self._in_eof = True
self._buffer.flush()
def _seek(self, offset, whence):
# Ensure we've read enough to simply do the seek on the buffer
if self._check_mode("r") and not self._in_eof:
if whence == 0:
if offset > self._in_pos:
self._read_rest()
if whence == 1:
if self._buffer.tell() + offset > self._in_pos:
self._read_rest()
if whence == 2:
self._read_rest()
# Then just do it on the buffer...
self._buffer.seek(offset, whence)
def _tell(self):
return self._buffer.tell()
def _truncate(self, size):
if self._check_mode("r") and not self._in_eof:
if size > self._in_pos:
self._read_rest()
self._in_eof = True
try:
self._buffer.truncate(size)
except TypeError:
et, ev, tb = sys.exc_info()
# SpooledTemporaryFile.truncate() doesn't accept size paramter.
try:
self._buffer._file.truncate(size)
except Exception:
raise et, ev, tb
# StringIO objects don't truncate to larger size correctly.
if hasattr(self._buffer, "_file"):
_file = self._buffer._file
if hasattr(_file, "getvalue"):
if len(_file.getvalue()) != size:
curpos = _file.tell()
_file.seek(0, 2)
_file.write("\x00" * (size - len(_file.getvalue())))
_file.seek(curpos)
self._was_truncated = True
def _read_rest(self):
"""Read the rest of the input stream."""
if self._in_eof:
return
pos = self._buffer.tell()
self._buffer.seek(0, 2)
data = self._fileobj.read(self._bufsize)
while data:
self._in_pos += len(data)
self._buffer.write(data)
data = self._fileobj.read(self._bufsize)
self._in_eof = True
self._buffer.flush()
self._buffer.seek(pos)
class Buffer(FileWrapper):
"""Class implementing buffereing of input and output streams.
This class uses a separate buffer file to hold the contents of the
underlying file while they are being manipulated. As data is read
it is duplicated into the buffer, and data is written from the buffer
back to the file on close.
"""
def __init__(self,fileobj,mode=None,max_size_in_memory=1024*8):
"""Buffered file wrapper constructor."""
self._buffer = SpooledTemporaryFile(max_size=max_size_in_memory)
self._in_eof = False
self._in_pos = 0
super(Buffer,self).__init__(fileobj,mode)
def _buffer_chunks(self):
chunk = self._buffer.read(16*1024)
if chunk == "":
yield chunk
else:
while chunk != "":
yield chunk
chunk = self._buffer.read(16*1024)
def _write_out_buffer(self):
if self._check_mode("r"):
self._read_rest()
if "a" in self.mode:
self._buffer.seek(self._in_pos)
self._fileobj.seek(self._in_pos)
else:
self._fileobj.seek(0)
self._buffer.seek(0)
else:
self._buffer.seek(0)
for chunk in self._buffer_chunks():
self._fileobj.write(chunk)
def flush(self):
# flush the buffer; we only write to the underlying file on close
self._buffer.flush()
def close(self):
if self.closed:
return
if self._check_mode("w"):
self._write_out_buffer()
super(Buffer,self).close()
self._buffer.close()
def _read(self,sizehint=-1):
# First return any data available from the buffer.
# Since we don't flush the buffer after every write, certain OSes
# (guess which!) will happy read junk data from the end of it.
# Instead, we explicitly read only up to self._in_pos.
if not self._in_eof:
buffered_size = self._in_pos - self._buffer.tell()
if sizehint >= 0:
buffered_size = min(sizehint,buffered_size)
else:
buffered_size = sizehint
data = self._buffer.read(buffered_size)
if data != "":
return data
# Then look for more data in the underlying file
if self._in_eof:
return None
data = self._fileobj.read(sizehint)
self._in_pos += len(data)
self._buffer.write(data)
if sizehint < 0 or len(data) < sizehint:
self._in_eof = True
self._buffer.flush()
return data
def _write(self,data,flushing=False):
self._buffer.write(data)
if self._check_mode("r") and not self._in_eof:
diff = self._buffer.tell() - self._in_pos
if diff > 0:
junk = self._fileobj.read(diff)
self._in_pos += len(junk)
if len(junk) < diff:
self._in_eof = True
self._buffer.flush()
def _seek(self,offset,whence):
# Ensure we've read enough to simply do the seek on the buffer
if self._check_mode("r") and not self._in_eof:
if whence == 0:
if offset > self._in_pos:
self._read_rest()
if whence == 1:
if self._buffer.tell() + offset > self._in_pos:
self._read_rest()
if whence == 2:
self._read_rest()
# Then just do it on the buffer...
self._buffer.seek(offset,whence)
def _tell(self):
return self._buffer.tell()
def _read_rest(self):
"""Read the rest of the input stream."""
if self._in_eof:
return
pos = self._buffer.tell()
self._buffer.seek(0,2)
data = self._fileobj.read(self._bufsize)
while data:
self._in_pos += len(data)
self._buffer.write(data)
data = self._fileobj.read(self._bufsize)
self._in_eof = True
self._buffer.flush()
self._buffer.seek(pos)
print(len(manyCaptions), ' records found to be vectorized..')
with open('vocab_from_allEnglish_captions_and_some_texts.pickle', 'rb') as f:
v = pickle.load(f)
print('length of vocabulary dictionary used is: ', len(v))
vectorizer = TfidfVectorizer(tokenizer=tokenizeText,
ngram_range=(1, 1),
vocabulary=v)
pipe = Pipeline([('cleanText', CleanTextTransformer()),
('vectorizer', vectorizer)])
timer1.timer()
for item in manyCaptions:
document = [item['captionsText']]
captionsID = item['id']
p = pipe.fit_transform(document)
f = SpooledTemporaryFile(max_size=1000000000)
scipyio.mmwrite(f, p[0])
f.seek(0) # important line..
fileContent = f.read()
with connection.cursor() as cursor:
sql = """UPDATE captions SET tfidfVector=%s WHERE id=%s"""
cursor.execute(sql, (fileContent, captionsID))
connection.commit()
connection.close()
timer1.timer()
