def generate_thumbnail(self, content):
content = file_from_content(content)
uploaded_image = Image.open(content)
if max(uploaded_image.size) >= self.max_size:
uploaded_image.thumbnail((self.max_size, self.max_size),
Image.BILINEAR)
content = SpooledTemporaryFile(INMEMORY_FILESIZE)
uploaded_image.save(content, uploaded_image.format)
content.seek(0)
thumbnail = uploaded_image.copy()
thumbnail.thumbnail(self.thumbnail_size, Image.ANTIALIAS)
thumbnail = thumbnail.convert('RGBA')
thumbnail.format = self.thumbnail_format
output = SpooledTemporaryFile(INMEMORY_FILESIZE)
thumbnail.save(output, self.thumbnail_format)
output.seek(0)
thumb_path, thumb_id = self.store_content(
output, 'thumb.%s' % self.thumbnail_format.lower())
self['thumb_id'] = thumb_id
self['thumb_path'] = thumb_path
thumbnail_file = self.thumb_file
self['_thumb_public_url'] = thumbnail_file.public_url
content.close()
def runTest(path,connectivity='Native',location='firefox'):
#Note that timeout is a global maxiumum and includes queuing time!
#it does not relate to how long the test runs
#TODO Error Handling
args = [
'webpagetest',
'test', path,
'--server', server,
'--key', key,
'--location', location,
'--runs', '1',
'--connectivity', connectivity,
'--label',path,
'--keepua', #Don't change the useragent to indicate this is a bot
'--first', #Don't try for a repeat view
'--poll','5' #How frequently to poll the web server for the result
#,'--timeout',str(timeout)
]
outT = SpooledTemporaryFile(mode='w+') #We can specify the size of the memory buffer here if we need.
#Stops us hitting the buffer limit if use pipe.
#cmd = ""
#for arg in args:
# cmd = cmd + arg + ' '
#print(cmd)
result = run(args,stdout=outT,bufsize=4096,check=True)
outT.seek(0) #Have to return to the start of the file to read it.
result = outT.read()
outT.close()
output = loads(result) #String to JSON
return output
def generate_thumbnail(self, content):
content = file_from_content(content)
try:
uploaded_image = Image.open(content)
except Exception:
flask.abort(400, ValidationError(
{
'message': 'File Format',
'object': {"error": "Format Incorrect"},
}
))
if max(uploaded_image.size) >= self.max_size:
uploaded_image.thumbnail((self.max_size, self.max_size), Image.BILINEAR)
content = SpooledTemporaryFile(INMEMORY_FILESIZE)
uploaded_image.save(content, uploaded_image.format)
content.seek(0)
thumbnail = uploaded_image.copy()
thumbnail.thumbnail(self.thumbnail_size, Image.ANTIALIAS)
thumbnail = thumbnail.convert('RGBA')
thumbnail.format = self.thumbnail_format
output = SpooledTemporaryFile(INMEMORY_FILESIZE)
thumbnail.save(output, self.thumbnail_format)
output.seek(0)
thumb_path, thumb_id = self.store_content(output,
'thumb.%s' % self.thumbnail_format.lower())
self['thumb_id'] = thumb_id
self['thumb_path'] = thumb_path
thumbnail_file = self.thumb_file
self['_thumb_public_url'] = thumbnail_file.public_url
content.close()
def _do_execute_direct(self, code):
shell = builtins.__xonsh_shell__
env = builtins.__xonsh_env__
out = io.StringIO()
err = io.StringIO()
enc = env.get('XONSH_ENCODING')
out = SpooledTemporaryFile(max_size=MAX_SIZE,
mode='w+t',
encoding=enc,
newline='\n')
err = SpooledTemporaryFile(max_size=MAX_SIZE,
mode='w+t',
encoding=enc,
newline='\n')
try:
with redirect_stdout(out), redirect_stderr(err), \
swap(builtins, '__xonsh_stdout_uncaptured__', out), \
swap(builtins, '__xonsh_stderr_uncaptured__', err), \
env.swap({'XONSH_STORE_STDOUT': False}):
shell.default(code)
interrupted = False
except KeyboardInterrupt:
interrupted = True
output, error = '', ''
if out.tell() > 0:
out.seek(0)
output = out.read()
if err.tell() > 0:
err.seek(0)
error = err.read()
out.close()
err.close()
return output, error, interrupted
class GoogleCloudFile(File):
def __init__(self, name, mode, storage):
self.name = name
self.mime_type = mimetypes.guess_type(name)[0]
self._mode = mode
self._storage = storage
# NOTE(mattrobenolt): This is the same change in behavior as in
# the s3 backend. We're opting now to load the file
# or metadata at this step. This means we won't actually
# know a file doesn't exist until we try to read it.
self.blob = FancyBlob(storage.download_url, self.name, storage.bucket)
self._file = None
self._is_dirty = False
@property
def size(self):
return self.blob.size
def _get_file(self):
if self._file is None:
with metrics.timer('filestore.read', instance='gcs'):
self._file = SpooledTemporaryFile(
max_size=self._storage.max_memory_size,
suffix=".GSStorageFile",
dir=None,
)
if 'r' in self._mode:
self._is_dirty = False
self.blob.download_to_file(self._file)
self._file.seek(0)
return self._file
def _set_file(self, value):
self._file = value
file = property(_get_file, _set_file)
def read(self, num_bytes=None):
if 'r' not in self._mode:
raise AttributeError("File was not opened in read mode.")
if num_bytes is None:
num_bytes = -1
return super(GoogleCloudFile, self).read(num_bytes)
def write(self, content):
if 'w' not in self._mode:
raise AttributeError("File was not opened in write mode.")
self._is_dirty = True
return super(GoogleCloudFile, self).write(force_bytes(content))
def close(self):
if self._file is not None:
if self._is_dirty:
self.file.seek(0)
self.blob.upload_from_file(self.file,
content_type=self.mime_type)
self._file.close()
self._file = None
def close(self):
"""Send the change to the DFS, and close the file."""
self.flush()
if 'c' not in self.mode:
SpooledTemporaryFile.close(self)
def close(self):
"""Send the change to the DFS, and close the file."""
self.flush()
if 'c' not in self.mode:
SpooledTemporaryFile.close(self)
class GoogleCloudFile(File):
def __init__(self, name, mode, storage):
self.name = name
self.mime_type = mimetypes.guess_type(name)[0]
self._mode = mode
self._storage = storage
self.blob = storage.bucket.get_blob(name)
if not self.blob and 'w' in mode:
self.blob = Blob(self.name,
storage.bucket,
chunk_size=storage.blob_chunk_size)
self._file = None
self._is_dirty = False
@property
def size(self):
return self.blob.size
def _get_file(self):
if self._file is None:
self._file = SpooledTemporaryFile(
max_size=self._storage.max_memory_size,
suffix=".GSStorageFile",
dir=setting("FILE_UPLOAD_TEMP_DIR"))
if 'r' in self._mode:
self._is_dirty = False
self.blob.download_to_file(self._file)
self._file.seek(0)
return self._file
def _set_file(self, value):
self._file = value
file = property(_get_file, _set_file)
def read(self, num_bytes=None):
if 'r' not in self._mode:
raise AttributeError("File was not opened in read mode.")
if num_bytes is None:
num_bytes = -1
return super().read(num_bytes)
def write(self, content):
if 'w' not in self._mode:
raise AttributeError("File was not opened in write mode.")
self._is_dirty = True
return super().write(force_bytes(content))
def close(self):
if self._file is not None:
if self._is_dirty:
self.blob.upload_from_file(
self.file,
rewind=True,
content_type=self.mime_type,
predefined_acl=self._storage.default_acl)
self._file.close()
self._file = None
def _do_execute_direct(self, code):
shell = builtins.__xonsh_shell__
env = builtins.__xonsh_env__
out = io.StringIO()
err = io.StringIO()
enc = env.get('XONSH_ENCODING')
out = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t',
encoding=enc, newline='\n')
err = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t',
encoding=enc, newline='\n')
try:
with redirect_stdout(out), redirect_stderr(err), \
swap(builtins, '__xonsh_stdout_uncaptured__', out), \
swap(builtins, '__xonsh_stderr_uncaptured__', err), \
env.swap({'XONSH_STORE_STDOUT': False}):
shell.default(code)
interrupted = False
except KeyboardInterrupt:
interrupted = True
output, error = '', ''
if out.tell() > 0:
out.seek(0)
output = out.read()
if err.tell() > 0:
err.seek(0)
error = err.read()
out.close()
err.close()
return output, error, interrupted
def archive_to_repo(archive_path, repo, archive_type="tar"):
"""Downloads a archive from the specified path,
extracts it into the repo's directory, commits
any changes from the previous version and pushes it
to github!!!!"""
# Download the tarball and stick it in a tempfile
r = requests.get(archive_path)
tmp = SpooledTemporaryFile()
tmp.write(r.content)
tmp.seek(0)
# Open the tempfile contents as an actual tarball
if archive_type == "tar":
archive = tarfile.open(fileobj=tmp)
elif archive_type == "zip":
archive = zipfile.open(tmp)
else:
raise ValueError("Unrecognized Archive Type")
# Clear working files
clear_working_dir(repo.working_dir)
# Extract to the repo path
archive.extract(repo.working_dir)
# Add and commit everything!
try:
repo.git.add(".", A=True)
repo.git.commit(m="New archive version")
except:
pass # May be that there was nothing new to commit
# Cleanup, outta here!
archive.close()
tmp.close()
def submitTest(job,server,key):
location = rowToLocation(job)
print(location +' '+job['script'])
args = [
'webpagetest',
'test', job['script'],
'--server', server,
'--key', key,
'--location', location,
'--runs', '1',
'--connectivity', 'Native',
'--label',job['script'],
'--keepua', #Don't change the useragent to indicate this is a bot
'--first', #Don't try for a repeat view
]
outT = SpooledTemporaryFile(mode='w+') #We can specify the size of the memory buffer here if we need.
#Stops us hitting the buffer limit if use pipe.
#cmd = ""
#for arg in args:
# cmd = cmd + arg + ' '
#print(cmd)
result = run(args,stdout=outT,bufsize=4096,check=True)
outT.seek(0) #Have to return to the start of the file to read it.
result = outT.read()
outT.close()
output = loads(result) #String to JSON
return output
def generate_thumbnail(self, content):
content = file_from_content(content)
uploaded_image = Image.open(content)
if max(uploaded_image.size) >= self.max_size:
uploaded_image.thumbnail((self.max_size, self.max_size), Image.BILINEAR)
content = SpooledTemporaryFile(INMEMORY_FILESIZE)
uploaded_image.save(content, uploaded_image.format)
content.seek(0)
thumbnail = uploaded_image.copy()
thumbnail.thumbnail(self.thumbnail_size, Image.ANTIALIAS)
thumbnail = thumbnail.convert('RGBA')
thumbnail.format = self.thumbnail_format
output = SpooledTemporaryFile(INMEMORY_FILESIZE)
thumbnail.save(output, self.thumbnail_format)
output.seek(0)
thumb_path, thumb_id = self.store_content(output,
'thumb.%s' % self.thumbnail_format.lower())
self['thumb_id'] = thumb_id
self['thumb_path'] = thumb_path
thumbnail_file = self.thumb_file
self['_thumb_public_url'] = thumbnail_file.public_url
content.close()
class GoogleCloudFile(File):
def __init__(self, name, mode, storage):
self.name = name
self.mime_type = mimetypes.guess_type(name)[0]
self._mode = mode
self._storage = storage
# NOTE(mattrobenolt): This is the same change in behavior as in
# the s3 backend. We're opting now to load the file
# or metadata at this step. This means we won't actually
# know a file doesn't exist until we try to read it.
self.blob = FancyBlob(storage.download_url, self.name, storage.bucket)
self._file = None
self._is_dirty = False
@property
def size(self):
return self.blob.size
def _get_file(self):
if self._file is None:
with metrics.timer('filestore.read', instance='gcs'):
self._file = SpooledTemporaryFile(
max_size=self._storage.max_memory_size,
suffix=".GSStorageFile",
dir=None,
)
if 'r' in self._mode:
self._is_dirty = False
self.blob.download_to_file(self._file)
self._file.seek(0)
return self._file
def _set_file(self, value):
self._file = value
file = property(_get_file, _set_file)
def read(self, num_bytes=None):
if 'r' not in self._mode:
raise AttributeError("File was not opened in read mode.")
if num_bytes is None:
num_bytes = -1
return super(GoogleCloudFile, self).read(num_bytes)
def write(self, content):
if 'w' not in self._mode:
raise AttributeError("File was not opened in write mode.")
self._is_dirty = True
return super(GoogleCloudFile, self).write(force_bytes(content))
def close(self):
if self._file is not None:
if self._is_dirty:
self.file.seek(0)
self.blob.upload_from_file(self.file, content_type=self.mime_type)
self._file.close()
self._file = None
def _save_content(self, obj, content, parameters):
content.seek(0, os.SEEK_SET)
content_autoclose = SpooledTemporaryFile()
content_autoclose.write(content.read())
super(CustomS3Boto3Storage,
self)._save_content(obj, content_autoclose, parameters)
if not content_autoclose.closed:
content_autoclose.close()
def ex(cmd):
"""Shell out a subprocess and return what it writes to stdout as a string"""
in_mem_file = SpooledTemporaryFile(max_size=2048, mode="r+")
subprocess.check_call(cmd, shell=True, stdout=in_mem_file)
in_mem_file.seek(0)
stdout = in_mem_file.read()
in_mem_file.close()
return stdout
def _shell_command(self, cmd): """Shell out a subprocess and return what it writes to stdout as a string""" in_mem_file = SpooledTemporaryFile(max_size=2048, mode="r+") check_call(cmd, shell=True, stdout=in_mem_file) in_mem_file.seek(0) stdout = in_mem_file.read() in_mem_file.close() return stdout
def image(type, spec=' ', ext='png'):
# Parameters for `suml`.
import suml.common
import optparse
options = optparse.Values(({
'scruffy':
True,
'png':
ext == 'png',
'svg':
ext == 'svg' or ext == 'pdf',
'font':
os.getenv('SCRUFFY_FONT', suml.common.defaultScruffyFont()),
'shadow':
False,
}))
from tempfile import SpooledTemporaryFile
fout = SpooledTemporaryFile()
# Execute Scruffy `suml`.
if type == 'class':
suml.yuml2dot.transform(spec, fout, options)
elif type == 'sequence':
suml.suml2pic.transform(spec, fout, options)
else:
return HTTPError(404, 'Unhandled diagram type.')
# Retrieve the data generated.
fout.seek(0)
data = fout.read()
fout.close()
# Convert SVG to PDF?
if ext == 'pdf':
# Load SVG file.
doc = xml.dom.expatbuilder.parseString(data)
# Convert to a RLG drawing
svg_renderer = svglib.svglib.SvgRenderer()
svg_renderer.render(doc.documentElement)
drawing = svg_renderer.finish()
# Generate PDF.
data = reportlab.graphics.renderPDF.drawToString(drawing)
# Server the generated image.
if ext == 'png':
response.content_type = 'image/png'
elif ext == 'svg':
response.content_type = 'image/svg+xml'
elif ext == 'pdf':
response.content_type = 'application/pdf'
else:
return HTTPError(500, 'Unhandled extension type.')
return data
class GoogleCloudFile(File):
def __init__(self, name, mode, storage):
self.name = name
self.mime_type = mimetypes.guess_type(name)[0]
self._mode = mode
self._storage = storage
self.blob = storage.bucket.get_blob(name)
if not self.blob and 'w' in mode:
self.blob = Blob(self.name, storage.bucket)
self._file = None
self._is_dirty = False
@property
def size(self):
return self.blob.size
def _get_file(self):
if self._file is None:
self._file = SpooledTemporaryFile(
max_size=self._storage.max_memory_size,
suffix=".GSStorageFile",
dir=setting("FILE_UPLOAD_TEMP_DIR", None)
)
if 'r' in self._mode:
self._is_dirty = False
self.blob.download_to_file(self._file)
self._file.seek(0)
return self._file
def _set_file(self, value):
self._file = value
file = property(_get_file, _set_file)
def read(self, num_bytes=None):
if 'r' not in self._mode:
raise AttributeError("File was not opened in read mode.")
if num_bytes is None:
num_bytes = -1
return super(GoogleCloudFile, self).read(num_bytes)
def write(self, content):
if 'w' not in self._mode:
raise AttributeError("File was not opened in write mode.")
self._is_dirty = True
return super(GoogleCloudFile, self).write(force_bytes(content))
def close(self):
if self._file is not None:
if self._is_dirty:
self.file.seek(0)
self.blob.upload_from_file(self.file, content_type=self.mime_type)
self._file.close()
self._file = None
def close(self):
"""On close, seek to 0 and write the data via the API, then close()
for realz
"""
logger.debug("close() called on WriteBuffer")
self.seek(0)
logger.debug("Attempting to create file at dir_path %s with name %s" %
(self.path, self.filename))
self.fs.rc.fs.write_file(self, self.fullpath)
SpooledTemporaryFile.close(self) # old-style class!
def getJSON(id, server):
args = ['webpagetest', 'results', id, '--server', server]
outT = SpooledTemporaryFile(
mode='w+'
) #We can specify the size of the memory buffer here if we need.
result = run(args, stdout=outT, bufsize=4096, check=True)
outT.seek(0) #Have to return to the start of the file to read it.
result = outT.read()
outT.close()
output = loads(result) #String to JSON
return output
class GoogleCloudFile(File):
def __init__(self, name, mode, storage, buffer_size=None):
self.name = name
self._mode = mode
self._storage = storage
self.blob = Blob(self.name, storage.bucket)
self._file = None
self._is_dirty = False
@property
def size(self):
return self.blob.size
def _get_file(self):
if self._file is None:
self._file = SpooledTemporaryFile(
max_size=self._storage.max_memory_size,
suffix=".GSStorageFile",
dir=setting("FILE_UPLOAD_TEMP_DIR", None))
if 'r' in self._mode:
self._is_dirty = False
self.blob.download_to_file(self._file)
self._file.seek(0)
return self._file
def _set_file(self, value):
self._file = value
file = property(_get_file, _set_file)
def read(self, *args, **kwargs):
if 'r' not in self._mode:
raise AttributeError("File was not opened in read mode.")
return super(GoogleCloudFile, self).read(*args, **kwargs)
def write(self, content, *args, **kwargs):
if 'w' not in self._mode:
raise AttributeError("File was not opened in write mode.")
self._is_dirty = True
return super(GoogleCloudFile, self).write(force_bytes(content), *args,
**kwargs)
def close(self):
if self._file is not None:
if self._is_dirty:
self.file.seek(0)
content_type, _ = mimetypes.guess_type(self.name)
content_type = getattr(self.file, 'content_type', content_type)
size = getattr(self.file, 'size')
self.blob.upload_from_file(self.file,
content_type=content_type,
size=size)
self._file.close()
self._file = None
class FTPHelper:
"""Base FTP helper class"""
def __init__(self, ftp, name, mode):
self.ftp = ftp
self.name = name
self.mode = mode
self.file = SpooledTemporaryFile(64 * 1024)
def close(self):
"""Close temporary file"""
self.file.close()
class GCPBlob(File):
def __init__( # noqa missed call to super
self,
file: Blob,
name: Optional[str] = None,
mode: Optional[str] = "rb") -> None:
self.name = name or file.name
self.mime_type = mimetypes.guess_type(self.name)[0]
self.blob = file
self._mode = mode
self._file = None
self._is_dirty = False
@property
def size(self) -> int:
return self.blob.size
def _get_file(self) -> SpooledTemporaryFile:
if self._file is None:
self._file = SpooledTemporaryFile(suffix=".GCPStorageFile", )
if 'r' in self._mode:
self._is_dirty = False
self.blob.download_to_file(self._file)
self._file.seek(0)
return self._file
def _set_file(self, value):
self._file = value
file = property(_get_file, _set_file)
def read(self, num_bytes=None):
if 'r' not in self._mode:
raise AttributeError("File was not opened in read mode.")
if num_bytes is None:
num_bytes = -1
return super().read(num_bytes)
def write(self, content):
if 'w' not in self._mode:
raise AttributeError("File was not opened in write mode.")
self._is_dirty = True
return super().write(force_bytes(content))
def close(self) -> None:
if self._file is not None:
if self._is_dirty:
self.blob.upload_from_file(self.file,
rewind=True,
content_type=self.mime_type)
self._file.close()
self._file = None
class RemoteFileBuffer(object):
"""File-like object providing buffer for local file operations.
Instances of this class manage a local tempfile buffer corresponding
to the contents of a remote file. All reads and writes happen locally,
with the content being copied to the remote file only on flush() or
close().
Instances of this class are returned by S3FS.open, but it is desgined
to be usable by any FS subclass that manages remote files.
"""
def __init__(self,fs,path,mode):
self.file = TempFile()
self.fs = fs
self.path = path
self.mode = mode
def __del__(self):
if not self.closed:
self.close()
# This is lifted straight from the stdlib's tempfile.py
def __getattr__(self,name):
file = self.__dict__['file']
a = getattr(file, name)
if not issubclass(type(a), type(0)):
setattr(self, name, a)
return a
def __enter__(self):
self.file.__enter__()
return self
def __exit__(self,exc,value,tb):
self.close()
return False
def __iter__(self):
return iter(self.file)
def flush(self):
self.file.flush()
if "w" in self.mode or "a" in self.mode or "+" in self.mode:
pos = self.file.tell()
self.file.seek(0)
self.fs.setcontents(self.path,self.file)
self.file.seek(pos)
def close(self):
if "w" in self.mode or "a" in self.mode or "+" in self.mode:
self.file.seek(0)
self.fs.setcontents(self.path,self.file)
self.file.close()
def upload_file(self,
user,
stream,
expected_size,
filename,
force_coll_name=''):
temp_file = None
logger.debug('Upload Begin')
logger.debug('Expected Size: ' + str(expected_size))
#is_anon = False
size_rem = user.get_size_remaining()
logger.debug('User Size Rem: ' + str(size_rem))
if size_rem < expected_size:
return {'error': 'out_of_space'}
if force_coll_name and not user.has_collection(force_coll_name):
#if is_anon:
# user.create_collection(force_coll, 'Temporary Collection')
#else:
#status = 'Collection {0} not found'.format(force_coll_name)
return {'error': 'no_such_collection'}
temp_file = SpooledTemporaryFile(max_size=BLOCK_SIZE)
stream = CacheingLimitReader(stream, expected_size, temp_file)
if filename.endswith('.har'):
stream, expected_size = self.har2warc(filename, stream)
temp_file.close()
temp_file = stream
infos = self.parse_uploaded(stream, expected_size)
total_size = temp_file.tell()
if total_size != expected_size:
return {
'error': 'incomplete_upload',
'expected': expected_size,
'actual': total_size
}
upload_id, upload_key = self._init_upload_status(user,
total_size,
1,
filename=filename)
return self.handle_upload(temp_file, upload_id, upload_key, infos,
filename, user, force_coll_name, total_size)
def upload_file(self, user, stream, expected_size, filename, force_coll_name=''):
"""Upload WARC archive.
:param User user: user
:param stream: file object
:param int expected_size: expected WARC archive size
:param str filename: WARC archive filename
:param str force_coll_name: name of collection to upload into
:returns: upload information
:rtype: dict
"""
temp_file = None
logger.debug('Upload Begin')
logger.debug('Expected Size: ' + str(expected_size))
#is_anon = False
size_rem = user.get_size_remaining()
logger.debug('User Size Rem: ' + str(size_rem))
if size_rem < expected_size:
return {'error': 'out_of_space'}
if force_coll_name and not user.has_collection(force_coll_name):
#if is_anon:
# user.create_collection(force_coll, 'Temporary Collection')
#else:
#status = 'Collection {0} not found'.format(force_coll_name)
return {'error': 'no_such_collection'}
temp_file = SpooledTemporaryFile(max_size=BLOCK_SIZE)
stream = CacheingLimitReader(stream, expected_size, temp_file)
if filename.endswith('.har'):
stream, expected_size = self.har2warc(filename, stream)
temp_file.close()
temp_file = stream
infos = self.parse_uploaded(stream, expected_size)
total_size = temp_file.tell()
if total_size != expected_size:
return {'error': 'incomplete_upload', 'expected': expected_size, 'actual': total_size}
upload_id, upload_key = self._init_upload_status(user, total_size, 1, filename=filename)
return self.handle_upload(temp_file, upload_id, upload_key, infos, filename,
user, force_coll_name, total_size)
def pipestring_process(cmd_string, stdin_string=''):
"""Pipe a python string to standard input for cmd_string
>>> pipestring_process('grep 2', '1\\n2\\n3\\n')
(0, '2\\n', '')
"""
f=SpooledTemporaryFile()
f.write(stdin_string)
f.seek(0)
results=process(cmd_string, stdin=f)
f.close()
return results
def __init__(self, path, filename, fs, max_size=WRITE_BUFFER_SIZE):
"""We need the path so we can write the buffered file to the API"""
SpooledTemporaryFile.__init__(self, max_size=max_size) # old-style!
self.path = path
self.filename = filename
self.fs = fs
self.fullpath = ''
try:
self.fullpath = self.create_file()
except RequestError, e:
SpooledTemporaryFile.close(self)
raise FilesystemError(str(e))
def do_execute(self, code, silent, store_history=True, user_expressions=None,
allow_stdin=False):
"""Execute user code."""
if len(code.strip()) == 0:
return {'status': 'ok', 'execution_count': self.execution_count,
'payload': [], 'user_expressions': {}}
env = builtins.__xonsh_env__
shell = builtins.__xonsh_shell__
hist = builtins.__xonsh_history__
enc = env.get('XONSH_ENCODING')
out = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t',
encoding=enc, newline='\n')
err = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t',
encoding=enc, newline='\n')
try:
with redirect_stdout(out), redirect_stderr(err), \
swap(builtins, '__xonsh_stdout_uncaptured__', out), \
swap(builtins, '__xonsh_stderr_uncaptured__', err), \
env.swap({'XONSH_STORE_STDOUT': False}):
shell.default(code)
interrupted = False
except KeyboardInterrupt:
interrupted = True
if not silent: # stdout response
if out.tell() > 0:
out.seek(0)
self._respond_in_chunks('stdout', out.read())
if err.tell() > 0:
err.seek(0)
self._respond_in_chunks('stderr', err.read())
if hasattr(builtins, '_') and builtins._ is not None:
# rely on sys.displayhook functionality
self._respond_in_chunks('stdout', pformat(builtins._))
builtins._ = None
if len(hist) > 0 and out.tell() == 0 and err.tell() == 0:
self._respond_in_chunks('stdout', hist.outs[-1])
out.close()
err.close()
if interrupted:
return {'status': 'abort', 'execution_count': self.execution_count}
rtn = 0 if len(hist) == 0 else hist.rtns[-1]
if 0 < rtn:
message = {'status': 'error', 'execution_count': self.execution_count,
'ename': '', 'evalue': str(rtn), 'traceback': []}
else:
message = {'status': 'ok', 'execution_count': self.execution_count,
'payload': [], 'user_expressions': {}}
return message
def image(type, spec=' ', ext='png'):
# Parameters for `suml`.
import suml.common
import optparse
options = optparse.Values(({
'scruffy': True,
'png': ext == 'png',
'svg': ext == 'svg' or ext == 'pdf',
'font': os.getenv('SCRUFFY_FONT', suml.common.defaultScruffyFont()),
'shadow': False,
}))
from tempfile import SpooledTemporaryFile
fout = SpooledTemporaryFile()
# Execute Scruffy `suml`.
if type == 'class':
suml.yuml2dot.transform(spec, fout, options)
elif type == 'sequence':
suml.suml2pic.transform(spec, fout, options)
else:
return HTTPError(404, 'Unhandled diagram type.')
# Retrieve the data generated.
fout.seek(0)
data = fout.read()
fout.close()
# Convert SVG to PDF?
if ext == 'pdf':
# Load SVG file.
doc = xml.dom.expatbuilder.parseString(data)
# Convert to a RLG drawing
svg_renderer = svglib.svglib.SvgRenderer()
svg_renderer.render(doc.documentElement)
drawing = svg_renderer.finish()
# Generate PDF.
data = reportlab.graphics.renderPDF.drawToString(drawing)
# Server the generated image.
if ext == 'png':
response.content_type = 'image/png'
elif ext == 'svg':
response.content_type = 'image/svg+xml'
elif ext == 'pdf':
response.content_type = 'application/pdf'
else:
return HTTPError(500, 'Unhandled extension type.')
return data
def do_execute(self, code, silent, store_history=True, user_expressions=None,
allow_stdin=False):
"""Execute user code."""
if len(code.strip()) == 0:
return {'status': 'ok', 'execution_count': self.execution_count,
'payload': [], 'user_expressions': {}}
env = builtins.__xonsh_env__
shell = builtins.__xonsh_shell__
hist = builtins.__xonsh_history__
enc = env.get('XONSH_ENCODING')
out = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t',
encoding=enc, newline='\n')
err = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t',
encoding=enc, newline='\n')
try:
with redirect_stdout(out), redirect_stderr(err), \
swap(builtins, '__xonsh_stdout_uncaptured__', out), \
swap(builtins, '__xonsh_stderr_uncaptured__', err), \
env.swap({'XONSH_STORE_STDOUT': False}):
shell.default(code)
interrupted = False
except KeyboardInterrupt:
interrupted = True
if not silent: # stdout response
if out.tell() > 0:
out.seek(0)
self._respond_in_chunks('stdout', out.read())
if err.tell() > 0:
err.seek(0)
self._respond_in_chunks('stderr', err.read())
if hasattr(builtins, '_') and builtins._ is not None:
# rely on sys.displayhook functionality
self._respond_in_chunks('stdout', pformat(builtins._))
builtins._ = None
if hist is not None and len(hist) > 0 and out.tell() == 0 and err.tell() == 0:
self._respond_in_chunks('stdout', hist.outs[-1])
out.close()
err.close()
if interrupted:
return {'status': 'abort', 'execution_count': self.execution_count}
rtn = 0 if (hist is None or len(hist) == 0) else hist.rtns[-1]
if 0 < rtn:
message = {'status': 'error', 'execution_count': self.execution_count,
'ename': '', 'evalue': str(rtn), 'traceback': []}
else:
message = {'status': 'ok', 'execution_count': self.execution_count,
'payload': [], 'user_expressions': {}}
return message
class BloomStringSet:
"""
BloomSet implements set membership in a fixed memory footprint,
but can reproduce the keys by streaming them to temporary files.
As it's based on a bloom filter, there is a risk of false positives,
which would cause some missing keyue, however the likelihood
of such is controllable through the parameters. Temporary files
are only created after they reach 5MB, otherwise stay in memory.
cardinality: the estimated maximum number of unique elements.
As one goes beyond this number, risk of collision
increases, but sloooooowly.
error_rate: the false positive rate you are comfortable with when
the cardinality number is reached.
"""
def __init__(self, cardinality=10**6, error_rate=10**-9):
self.bloom = BloomFilter(cardinality, error_rate)
self.file = SpooledTemporaryFile(max_size=(2**20) * 5, mode='w')
self.closed = False
def add(self, key):
if self.closed:
raise Exception("Cannot add new element after attempting to read")
if type(key) is not str:
raise Exception("Can only use string keys for now")
if key in self.bloom:
return False
self.bloom.add(key)
self.file.write(key + "\n")
def __contains__(self, key):
return key in self.bloom
def __iter__(self):
self.closed = True
self.file.seek(0)
return self
def __next__(self):
line = self.file.readline()
if line == '':
self.file.close()
raise StopIteration
return line.strip()
def __del__(self):
self.file.close()
class TempInput(object):
def __init__(self, inputstr):
self.inputstr = inputstr
def __enter__(self):
self.tempfile = SpooledTemporaryFile()
self.tempfile.write(self.inputstr)
self.tempfile.seek(0)
return self.tempfile
def __exit__(self, type_, value, traceback):
self.tempfile.close()
return False
def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False):
"""Execute user code."""
if len(code.strip()) == 0:
return {"status": "ok", "execution_count": self.execution_count, "payload": [], "user_expressions": {}}
env = builtins.__xonsh_env__
shell = builtins.__xonsh_shell__
hist = builtins.__xonsh_history__
enc = env.get("XONSH_ENCODING")
out = SpooledTemporaryFile(max_size=MAX_SIZE, mode="w+t", encoding=enc, newline="\n")
err = SpooledTemporaryFile(max_size=MAX_SIZE, mode="w+t", encoding=enc, newline="\n")
try:
with redirect_stdout(out), redirect_stderr(err), swap(builtins, "__xonsh_stdout_uncaptured__", out), swap(
builtins, "__xonsh_stderr_uncaptured__", err
), env.swap({"XONSH_STORE_STDOUT": False}):
shell.default(code)
interrupted = False
except KeyboardInterrupt:
interrupted = True
if not silent: # stdout response
if out.tell() > 0:
out.seek(0)
self._respond_in_chunks("stdout", out.read())
if err.tell() > 0:
err.seek(0)
self._respond_in_chunks("stderr", err.read())
if hasattr(builtins, "_") and builtins._ is not None:
# rely on sys.displayhook functionality
self._respond_in_chunks("stdout", pformat(builtins._))
builtins._ = None
if len(hist) > 0 and out.tell() == 0 and err.tell() == 0:
self._respond_in_chunks("stdout", hist.outs[-1])
out.close()
err.close()
if interrupted:
return {"status": "abort", "execution_count": self.execution_count}
rtn = 0 if len(hist) == 0 else hist.rtns[-1]
if 0 < rtn:
message = {
"status": "error",
"execution_count": self.execution_count,
"ename": "",
"evalue": str(rtn),
"traceback": [],
}
else:
message = {"status": "ok", "execution_count": self.execution_count, "payload": [], "user_expressions": {}}
return message
class TempInput(object):
def __init__(self, inputstr):
self.inputstr = inputstr
def __enter__(self):
self.tempfile = SpooledTemporaryFile()
self.tempfile.write(self.inputstr)
self.tempfile.seek(0)
return self.tempfile
def __exit__(self, type_, value, traceback):
self.tempfile.close()
return False
def checkFinished(id, server):
args = ['webpagetest', 'status', id, '--server', server]
outT = SpooledTemporaryFile(
mode='w+'
) #We can specify the size of the memory buffer here if we need.
result = run(args, stdout=outT, bufsize=4096, check=True)
outT.seek(0) #Have to return to the start of the file to read it.
result = outT.read()
outT.close()
output = loads(result) #String to JSON
if int(output['statusCode']) == 200:
return True
else:
return False
def csv_fn(pydata):
csvfile=SpooledTemporaryFile()
csvw=csv.writer(csvfile)
fieldnames=[]
for h in pydata:
for k in pydata[h]:
if k not in fieldnames:
fieldnames.append(k)
if showheader:
csvw.writerow(['system_name'] + fieldnames)
for system_name in pydata:
csvw.writerow([system_name] + [pydata[system_name].get(k, None) for k in fieldnames])
csvfile.seek(0)
results=csvfile.read()
csvfile.close()
return results
def _save_content(self, obj, content, parameters):
# Seek our content back to the start
content.seek(0, os.SEEK_SET)
# Create a temporary file that will write to disk after a specified size
content_autoclose = SpooledTemporaryFile()
# Write our original content into our copy that will be closed by boto3
content_autoclose.write(content.read())
# Upload the object which will auto close the content_autoclose instance
super(S3DefaultStorage, self)._save_content(obj, content_autoclose, parameters)
# Cleanup if this is fixed upstream our duplicate should always close
if not content_autoclose.closed:
content_autoclose.close()
class OrcCompressor(object):
def __init__(self):
self._buffer = SpooledTemporaryFile(mode='wb')
def compress(self, data: bytes):
self._buffer.write(data)
def flush(self):
if self._buffer:
self._buffer.seek(0)
df = pd.read_csv(self._buffer)
with SpooledTemporaryFile(mode='wb') as wfd:
df.to_parquet(wfd, engine='pyarrow')
wfd.seek(0)
return wfd.read()
def __del__(self):
self._buffer.close()
class InputStream(object):
"""
FCGI_STDIN or FCGI_DATA stream.
Uses temporary file to store received data once max_mem bytes
have been received.
"""
def __init__(self, max_mem=1024):
self._file = SpooledTemporaryFile(max_mem)
self._eof_received = Event()
def __del__(self):
self._file.close()
def feed(self, data):
if self._eof_received.is_set():
raise IOError('Feeding file beyond EOF mark')
if not data: # EOF mark
self._file.seek(0)
self._eof_received.set()
else:
if isinstance(data, six.text_type):
data = data.encode("ISO-8859-1")
self._file.write(data)
def __iter__(self):
self._eof_received.wait()
return iter(self._file)
def read(self, size=-1):
self._eof_received.wait()
return self._file.read(size)
def readline(self, size=-1):
self._eof_received.wait()
return self._file.readline(size)
def readlines(self, sizehint=0):
self._eof_received.wait()
return self._file.readlines(sizehint)
@property
def eof_received(self):
return self._eof_received.is_set()
class ParquetDecompressor(object):
def __init__(self, sep=b','):
self._buffer = SpooledTemporaryFile(mode='wb')
self._result = SpooledTemporaryFile(mode='wb')
self._sep = sep
self.eof = False
def decompress(self, data: bytes):
self._buffer.write(data)
return b''
def flush(self):
self._buffer.seek(0)
# TODO: to optimize it
df = pd.read_parquet(self._buffer, engine='pyarrow')
return df.to_csv(index=False).encode('utf-8')
def __del__(self):
self._buffer.close()
self._result.close()
def _save_content(self, obj, content, parameters):
"""
We create a clone of the content file as when this is passed to boto3 it wrongly closes
the file upon upload where as the storage backend expects it to still be open
"""
# Seek our content back to the start
content.seek(0, os.SEEK_SET)
# Create a temporary file that will write to disk after a specified size
content_autoclose = SpooledTemporaryFile()
# Write our original content into our copy that will be closed by boto3
content_autoclose.write(content.read())
# Upload the object which will auto close the content_autoclose instance
super(CustomS3Boto3Storage, self)._save_content(obj, content_autoclose, parameters)
# Cleanup if this is fixed upstream our duplicate should always close
if not content_autoclose.closed:
content_autoclose.close()
class TempInput(object):
""" Acts as STDIN for a Popen process.
Initialize with the string you want to send on startup.
Enoding can be set with the optional 'encoding' arg.
"""
def __init__(self, inputstr, encoding=None):
self.encoding = encoding or 'utf-8'
self.inputstr = inputstr.encode(self.encoding)
def __enter__(self):
self.tempfile = SpooledTemporaryFile()
self.tempfile.write(self.inputstr)
self.tempfile.seek(0)
return self.tempfile
def __exit__(self, type_, value, traceback):
self.tempfile.close()
return False
def _save_content(self, obj, content, parameters):
"""
We create a clone of the content file as when this is passed to boto3 it wrongly closes
the file upon upload where as the storage backend expects it to still be open
"""
# Seek our content back to the start
content.seek(0, os.SEEK_SET)
# Create a temporary file that will write to disk after a specified size
content_autoclose = SpooledTemporaryFile()
# Write our original content into our copy that will be closed by boto3
content_autoclose.write(content.read())
# Upload the object which will auto close the content_autoclose instance
super(PrivateMediaStorage, self)._save_content(obj, content_autoclose, parameters)
# Cleanup if this is fixed upstream our duplicate should always close
if not content_autoclose.closed:
content_autoclose.close()
def upload_to_s3(path, file, bucket_name=None):
s3 = get_boto_client()
bucket_name = bucket_name or current_app.config["AWS_S3_BUCKET"]
# Need to clone the file because boto3 autocloses the file after uploading to s3,
# preventing further use of the uploaded file.
file_s3 = SpooledTemporaryFile()
file_s3.write(file.read())
file_s3.seek(0)
s3.upload_fileobj(
file_s3,
bucket_name,
path,
ExtraArgs={"ACL": "public-read", "ContentType": file.content_type},
)
if not file_s3.closed:
file_s3.close()
return bucket_name, path
def download_poster(image_url, out_dir):
print(" [ Api ] Downloading cover art :")
buffer = SpooledTemporaryFile(max_size=1e9)
r = get(image_url, stream=True)
if r.status_code == 200:
downloaded = 0
filesize = int(r.headers['content-length'])
for chunk in r.iter_content(chunk_size=1024):
downloaded += len(chunk)
buffer.write(chunk)
print_progress_bar(downloaded, filesize)
time.sleep(0.007)
# print(downloaded/filesize)
buffer.seek(0)
print()
if Path.joinpath(out_dir, 'image.jpg').exists():
os.remove(Path.joinpath(out_dir, 'image.jpg'))
i = Image.open(BytesIO(buffer.read()))
i.save(os.path.join(out_dir, 'image.jpg'), quality=100)
buffer.close()
class LLDB:
def _parseStackTrace(self, jibberish):
not_jibberish = re.findall(r'\(lldb\) bt(.*)\(lldb\)', jibberish, re.DOTALL)
if len(not_jibberish) != 0:
return not_jibberish[0].replace(' frame ', '')
else:
return 'Stack Trace failed:', jibberish
def _waitForResponse(self, wait=True):
while wait:
self.lldb_stdout.seek(self.last_position)
for line in self.lldb_stdout:
if line == '(lldb) ':
self.last_position = self.lldb_stdout.tell()
return True
time.sleep(0.05)
time.sleep(0.05)
return True
def getStackTrace(self, pid):
lldb_commands = [ 'attach -p ' + pid + '\n', 'bt\n', 'quit\n', 'Y\n' ]
self.lldb_stdout = SpooledTemporaryFile()
self.last_position = 0
lldb_process = subprocess.Popen(['lldb', '-x'], stdin=subprocess.PIPE, stdout=self.lldb_stdout, stderr=self.lldb_stdout)
while lldb_process.poll() == None:
for command in lldb_commands:
if command == lldb_commands[-1]:
lldb_commands = []
if self._waitForResponse(False):
# I have seen LLDB exit out from under us
try:
lldb_process.stdin.write(command)
except:
pass
elif self._waitForResponse():
lldb_process.stdin.write(command)
self.lldb_stdout.seek(0)
stack_trace = self._parseStackTrace(self.lldb_stdout.read())
self.lldb_stdout.close()
return stack_trace
class GDB:
def _parseStackTrace(self, jibberish):
not_jibberish = re.findall(r'\(gdb\) (#.*)\(gdb\)', jibberish, re.DOTALL)
if len(not_jibberish) != 0:
return not_jibberish[0]
else:
return 'Stack Trace failed:', jibberish
def _waitForResponse(self, wait=True):
while wait:
self.gdb_stdout.seek(self.last_position)
for line in self.gdb_stdout:
if line == '(gdb) ':
self.last_position = self.gdb_stdout.tell()
return True
time.sleep(0.05)
time.sleep(0.05)
return True
def getStackTrace(self, pid):
gdb_commands = [ 'attach ' + pid + '\n', 'set verbose off\n', 'thread\n', 'apply\n', 'all\n', 'bt\n', 'quit\n', 'y\n' ]
self.gdb_stdout = SpooledTemporaryFile()
self.last_position = 0
gdb_process = subprocess.Popen(['gdb', '-nx'], stdin=subprocess.PIPE, stdout=self.gdb_stdout, stderr=self.gdb_stdout)
while gdb_process.poll() == None:
for command in gdb_commands:
if command == gdb_commands[-1]:
gdb_commands = []
elif self._waitForResponse():
# I have seen GDB exit out from under us
try:
gdb_process.stdin.write(command)
except:
pass
self.gdb_stdout.seek(0)
stack_trace = self._parseStackTrace(self.gdb_stdout.read())
self.gdb_stdout.close()
return stack_trace
class StartFinish(object):
def __init__(self):
pass
def start(self, args):
self.outFile = SpooledTemporaryFile()
self.errFile = SpooledTemporaryFile()
self.cmdline = list2cmdline(args)
print 'starting: ' + self.cmdline
self.process = Popen(args,
stderr=self.errFile, stdout=self.outFile, universal_newlines=False)
self.process_start = time()
def finish(self, timeout):
print 'finishing "' + self.cmdline + '"'
kill_time = self.process_start + timeout
self.process_end = time()
while self.process.poll() is None:
self.process_end = time()
if self.process_end < kill_time:
sleep(0.1)
else:
self.process.kill()
raise Exception('timeout "' + self.cmdline + '"')
# read temp streams from start
self.outFile.seek(0)
self.errFile.seek(0)
self.out = self.outFile.read()
self.err = self.errFile.read()
self.outFile.close()
self.errFile.close()
print 'out:', self.out
print 'err:', self.err
print 'returncode:', self.process.returncode
print 'elapsed:', self.process_end - self.process_start
return self.process.returncode
class Buffer(FileWrapper):
"""Class implementing buffering of input and output streams.
This class uses a separate buffer file to hold the contents of the
underlying file while they are being manipulated. As data is read
it is duplicated into the buffer, and data is written from the buffer
back to the file on close.
"""
def __init__(self, fileobj, mode=None, max_size_in_memory=1024 * 8):
"""Buffered file wrapper constructor."""
self._buffer = SpooledTemporaryFile(max_size=max_size_in_memory)
self._in_eof = False
self._in_pos = 0
self._was_truncated = False
super(Buffer, self).__init__(fileobj, mode)
def _buffer_size(self):
try:
return len(self._buffer.file.getvalue())
except AttributeError:
return os.fstat(self._buffer.fileno()).st_size
def _buffer_chunks(self):
chunk = self._buffer.read(16 * 1024)
if chunk == "":
yield chunk
else:
while chunk != "":
yield chunk
chunk = self._buffer.read(16 * 1024)
def _write_out_buffer(self):
if self._check_mode("r"):
self._read_rest()
if "a" in self.mode:
self._buffer.seek(self._in_pos)
self._fileobj.seek(self._in_pos)
else:
self._fileobj.seek(0)
self._buffer.seek(0)
else:
self._buffer.seek(0)
if self._was_truncated:
self._fileobj.truncate(0)
self._was_truncated = False
for chunk in self._buffer_chunks():
self._fileobj.write(chunk)
def flush(self):
# flush the buffer; we only write to the underlying file on close
self._buffer.flush()
def close(self):
if self.closed:
return
if self._check_mode("w"):
self._write_out_buffer()
super(Buffer, self).close()
self._buffer.close()
def _read(self, sizehint=-1):
# First return any data available from the buffer.
# Since we don't flush the buffer after every write, certain OSes
# (guess which!) will happily read junk data from the end of it.
# Instead, we explicitly read only up to self._in_pos.
if not self._in_eof:
buffered_size = self._in_pos - self._buffer.tell()
if sizehint >= 0:
buffered_size = min(sizehint, buffered_size)
else:
buffered_size = sizehint
data = self._buffer.read(buffered_size)
if data != "":
return data
# Then look for more data in the underlying file
if self._in_eof:
return None
data = self._fileobj.read(sizehint)
self._in_pos += len(data)
self._buffer.write(data)
if sizehint < 0 or len(data) < sizehint:
self._in_eof = True
self._buffer.flush()
return data
def _write(self, data, flushing=False):
self._buffer.write(data)
if self._check_mode("r") and not self._in_eof:
diff = self._buffer.tell() - self._in_pos
if diff > 0:
junk = self._fileobj.read(diff)
self._in_pos += len(junk)
if len(junk) < diff:
self._in_eof = True
self._buffer.flush()
def _seek(self, offset, whence):
# Ensure we've read enough to simply do the seek on the buffer
if self._check_mode("r") and not self._in_eof:
if whence == 0:
if offset > self._in_pos:
self._read_rest()
if whence == 1:
if self._buffer.tell() + offset > self._in_pos:
self._read_rest()
if whence == 2:
self._read_rest()
# Then just do it on the buffer...
self._buffer.seek(offset, whence)
def _tell(self):
return self._buffer.tell()
def _truncate(self, size):
if self._check_mode("r") and not self._in_eof:
if size > self._in_pos:
self._read_rest()
self._in_eof = True
try:
self._buffer.truncate(size)
except TypeError:
et, ev, tb = sys.exc_info()
# SpooledTemporaryFile.truncate() doesn't accept size paramter.
try:
self._buffer._file.truncate(size)
except Exception:
raise et, ev, tb
# StringIO objects don't truncate to larger size correctly.
if hasattr(self._buffer, "_file"):
_file = self._buffer._file
if hasattr(_file, "getvalue"):
if len(_file.getvalue()) != size:
curpos = _file.tell()
_file.seek(0, 2)
_file.write("\x00" * (size - len(_file.getvalue())))
_file.seek(curpos)
self._was_truncated = True
def _read_rest(self):
"""Read the rest of the input stream."""
if self._in_eof:
return
pos = self._buffer.tell()
self._buffer.seek(0, 2)
data = self._fileobj.read(self._bufsize)
while data:
self._in_pos += len(data)
self._buffer.write(data)
data = self._fileobj.read(self._bufsize)
self._in_eof = True
self._buffer.flush()
self._buffer.seek(pos)
class IncrementalWriter:
"""A streaming file writer for point clouds.
Using the IncrementalWriter with spooled temporary files, which are
only flushed to disk if they go above the given size, allows for
streaming points to disk even when the header is unknown in advance.
This allows some nice tricks, including splitting a point cloud into
multiple files in a single pass, without memory issues.
"""
# pylint:disable=too-few-public-methods
def __init__(self, filename: str, header: PlyHeader,
utm: UTM_Coord=None, buffer=2**22) -> None:
"""
Args:
filename: final place to save the file on disk.
source_fname: source file for the pointcloud; used to detect
file format for metadata etc.
buffer (int): The number of bytes to hold in RAM before flushing
the temporary file to disk. Default 1MB, which holds ~8300
points - enough for most objects but still practical to hold
thousands in memory. Set a smaller buffer for large forests.
"""
self.filename = filename
self.temp_storage = SpooledTemporaryFile(max_size=buffer, mode='w+b')
self.count = 0
self.utm = utm
self.header = header
# Always write in big-endian mode; only store type information
self.binary = struct.Struct('>' + header.form_str[1:])
def __call__(self, point) -> None:
"""Add a single point to this pointcloud, saving in binary format.
Args:
point (namedtuple): vertex attributes for the point, eg xyzrgba.
"""
self.temp_storage.write(self.binary.pack(*point))
self.count += 1
def __del__(self):
"""Flush data to disk and clean up."""
to_ply_types = {v: k for k, v in PLY_TYPES.items()}
properties = ['property {t} {n}'.format(t=t, n=n) for t, n in zip(
(to_ply_types[p] for p in self.header.form_str[1:]),
self.header.names)]
head = ['ply',
'format binary_big_endian 1.0',
'element vertex {}'.format(self.count),
'\n'.join(properties),
'end_header']
if self.utm is not None:
head.insert(-1, 'comment UTM x y zone north ' +
'{0.x} {0.y} {0.zone} {0.north}'.format(self.utm))
if not os.path.isdir(os.path.dirname(self.filename)):
os.makedirs(os.path.dirname(self.filename))
with open(self.filename, 'wb') as f:
f.write(('\n'.join(head) + '\n').encode('ascii'))
self.temp_storage.seek(0)
chunk = self.temp_storage.read(8192)
while chunk:
f.write(chunk)
chunk = self.temp_storage.read(8192)
self.temp_storage.close()
def runTest(test, testnum):
# test is a tuple of ( filename , usedb<bool> )
# filename should be a js file to run
# usedb is true if the test expects a mongod to be running
(path, usedb) = test
(ignore, ext) = os.path.splitext(path)
# the dbtests know how to format themselves nicely, we'll detect if we're running them and if
# so, we won't mess with the output
is_test_binary = False
if skipTest(path):
if quiet:
sys.stdout.write("skip %d %s\n" % (testnum, os.path.basename(path)))
sys.stdout.flush()
else:
print "skipping " + path
return
if file_of_commands_mode:
# smoke.py was invoked like "--mode files --from-file foo",
# so don't try to interpret the test path too much
if os.sys.platform == "win32":
argv = [path]
else:
argv = shlex.split(path)
path = argv[0]
# if the command is a python script, use the script name
if os.path.basename(path) in ('python', 'python.exe'):
path = argv[1]
elif ext == ".js":
argv = [shell_executable, "--port", mongod_port, '--authenticationMechanism', authMechanism]
if not usedb:
argv += ["--nodb"]
if small_oplog or small_oplog_rs:
argv += ["--eval", 'testingReplication = true;']
if use_ssl:
argv += ["--ssl",
"--sslPEMKeyFile", "jstests/libs/client.pem",
"--sslCAFile", "jstests/libs/ca.pem"]
argv += [path]
elif ext in ["", ".exe"]:
# Blech.
if os.path.basename(path) in ["test", "test.exe", "perftest", "perftest.exe"]:
argv = [path]
if os.path.basename(path) in ["test", "test.exe"]:
is_test_binary = True
# more blech
elif os.path.basename(path) in ['mongos', 'mongos.exe']:
argv = [path, "--test"]
else:
argv = [test_path and os.path.abspath(os.path.join(test_path, path)) or path,
"--port", mongod_port]
else:
raise Bug("fell off in extension case: %s" % path)
if keyFile:
f = open(keyFile, 'r')
keyFileData = re.sub(r'\s', '', f.read()) # Remove all whitespace
f.close()
os.chmod(keyFile, stat.S_IRUSR | stat.S_IWUSR)
else:
keyFileData = None
mongo_test_filename = os.path.basename(path)
if 'sharedclient' in path:
mongo_test_filename += "-sharedclient"
# sys.stdout.write() is more atomic than print, so using it prevents
# lines being interrupted by, e.g., child processes
if quiet and not is_test_binary:
vlog = tests_log
qlog = sys.stdout
tlog = sys.stderr
else:
vlog = sys.stdout
qlog = None
tlog = None
vlog.write(" *******************************************\n")
vlog.write(" Test : %s ...\n" % mongo_test_filename)
vlog.flush()
# FIXME: we don't handle the case where the subprocess
# hangs... that's bad.
if ( argv[0].endswith( 'mongo' ) or argv[0].endswith( 'mongo.exe' ) ) and not '--eval' in argv :
evalString = 'TestData = new Object();' + \
'TestData.testPath = "' + path + '";' + \
'TestData.testFile = "' + os.path.basename( path ) + '";' + \
'TestData.testName = "' + re.sub( ".js$", "", os.path.basename( path ) ) + '";' + \
'TestData.noJournal = ' + ternary( no_journal ) + ";" + \
'TestData.noJournalPrealloc = ' + ternary( no_preallocj ) + ";" + \
'TestData.auth = ' + ternary( auth ) + ";" + \
'TestData.keyFile = ' + ternary( keyFile , '"' + str(keyFile) + '"' , 'null' ) + ";" + \
'TestData.keyFileData = ' + ternary( keyFile , '"' + str(keyFileData) + '"' , 'null' ) + ";"
if os.sys.platform == "win32":
# double quotes in the evalString on windows; this
# prevents the backslashes from being removed when
# the shell (i.e. bash) evaluates this string. yuck.
evalString = evalString.replace('\\', '\\\\')
if auth and usedb:
evalString += 'jsTest.authenticate(db.getMongo());'
argv = argv + [ '--eval', evalString]
if argv[0].endswith( 'test' ) and no_preallocj :
argv = argv + [ '--nopreallocj' ]
vlog.write(" Command : %s\n" % ' '.join(argv))
vlog.write(" Date : %s\n" % datetime.now().ctime())
vlog.flush()
tempfile = SpooledTemporaryFile(max_size=16*1024*1024)
try:
os.environ['MONGO_TEST_FILENAME'] = mongo_test_filename
t1 = time.time()
r = call(buildlogger(argv), cwd=test_path,
# the dbtests know how to format their own output nicely
stdout=ternary(is_test_binary, vlog, tempfile))
t2 = time.time()
del os.environ['MONGO_TEST_FILENAME']
vlog.write(" %fms\n" % ((t2 - t1) * 1000))
vlog.flush()
if not is_test_binary:
tempfile.seek(0)
for line in tempfile:
vlog.write(line)
vlog.flush()
if quiet:
if r == 0:
qlog.write('ok %d %s\n' % (testnum, os.path.basename(path)))
else:
qlog.write('not ok %d %s # exit %d\n' % (testnum, os.path.basename(path), r))
qlog.flush()
if r != 0:
tempfile.seek(0)
for line in tempfile:
tlog.write(line)
tlog.flush()
if r != 0:
raise TestExitFailure(path, r)
finally:
tempfile.close()
if r != 0:
raise TestExitFailure(path, r)
if start_mongod:
try:
c = Connection(host="127.0.0.1", port=int(mongod_port), ssl=use_ssl)
except Exception,e:
print "Exception from pymongo: ", e
raise TestServerFailure(path)
def runTest(test):
# test is a tuple of ( filename , usedb<bool> )
# filename should be a js file to run
# usedb is true if the test expects a mongod to be running
(path, usedb) = test
(ignore, ext) = os.path.splitext(path)
if skipTest(path):
print "skipping " + path
return
if file_of_commands_mode:
# smoke.py was invoked like "--mode files --from-file foo",
# so don't try to interpret the test path too much
if os.sys.platform == "win32":
argv = [path]
else:
argv = shlex.split(path)
path = argv[0]
# if the command is a python script, use the script name
if os.path.basename(path) in ('python', 'python.exe'):
path = argv[1]
elif ext == ".js":
argv = [shell_executable, "--port", mongod_port]
if not usedb:
argv += ["--nodb"]
if small_oplog or small_oplog_rs:
argv += ["--eval", 'testingReplication = true;']
argv += [path]
elif ext in ["", ".exe"]:
# Blech.
if os.path.basename(path) in ["test", "test.exe", "perftest", "perftest.exe"]:
argv = [path]
# more blech
elif os.path.basename(path) in ['mongos', 'mongos.exe']:
argv = [path, "--test"]
else:
argv = [test_path and os.path.abspath(os.path.join(test_path, path)) or path,
"--port", mongod_port]
else:
raise Bug("fell off in extenstion case: %s" % path)
if keyFile:
f = open(keyFile, 'r')
keyFileData = re.sub(r'\s', '', f.read()) # Remove all whitespace
f.close()
else:
keyFileData = None
# sys.stdout.write() is more atomic than print, so using it prevents
# lines being interrupted by, e.g., child processes
if quiet:
vlog = tests_log
qlog = sys.stdout
else:
vlog = sys.stdout
qlog = None
vlog.write(" *******************************************\n")
vlog.write(" Test : %s ...\n" % os.path.basename(path))
vlog.flush()
# FIXME: we don't handle the case where the subprocess
# hangs... that's bad.
if ( argv[0].endswith( 'mongo' ) or argv[0].endswith( 'mongo.exe' ) ) and not '--eval' in argv :
evalString = 'TestData = new Object();' + \
'TestData.testPath = "' + path + '";' + \
'TestData.testFile = "' + os.path.basename( path ) + '";' + \
'TestData.testName = "' + re.sub( ".js$", "", os.path.basename( path ) ) + '";' + \
'TestData.noJournal = ' + ternary( no_journal ) + ";" + \
'TestData.noJournalPrealloc = ' + ternary( no_preallocj ) + ";" + \
'TestData.auth = ' + ternary( auth ) + ";" + \
'TestData.keyFile = ' + ternary( keyFile , '"' + str(keyFile) + '"' , 'null' ) + ";" + \
'TestData.keyFileData = ' + ternary( keyFile , '"' + str(keyFileData) + '"' , 'null' ) + ";"
if os.sys.platform == "win32":
# double quotes in the evalString on windows; this
# prevents the backslashes from being removed when
# the shell (i.e. bash) evaluates this string. yuck.
evalString = evalString.replace('\\', '\\\\')
if auth and usedb:
evalString += 'jsTest.authenticate(db.getMongo());'
argv = argv + [ '--eval', evalString]
if argv[0].endswith( 'test' ) and no_preallocj :
argv = argv + [ '--nopreallocj' ]
vlog.write(" Command : %s\n" % ' '.join(argv))
vlog.write(" Date : %s\n" % datetime.now().ctime())
vlog.flush()
tempfile = SpooledTemporaryFile(max_size=16*1024*1024)
try:
os.environ['MONGO_TEST_FILENAME'] = os.path.basename(path)
t1 = time.time()
r = call(buildlogger(argv), cwd=test_path, stdout=tempfile)
t2 = time.time()
del os.environ['MONGO_TEST_FILENAME']
vlog.write(" %fms\n" % ((t2 - t1) * 1000))
vlog.flush()
tempfile.seek(0)
for line in tempfile:
vlog.write(line)
vlog.flush()
if quiet:
qlog.write("%s%s : %s\n" % ((" " * max(0, 64 - len(os.path.basename(path)))),
os.path.basename(path),
ternary(r == 0, "ok", "fail")))
if r != 0:
tempfile.seek(0)
for line in tempfile:
qlog.write(line)
qlog.flush()
if r != 0:
raise TestExitFailure(path, r)
finally:
tempfile.close()
class Buffer(FileWrapper):
"""Class implementing buffereing of input and output streams.
This class uses a separate buffer file to hold the contents of the
underlying file while they are being manipulated. As data is read
it is duplicated into the buffer, and data is written from the buffer
back to the file on close.
"""
def __init__(self,fileobj,mode=None,max_size_in_memory=1024*8):
"""Buffered file wrapper constructor."""
self._buffer = SpooledTemporaryFile(max_size=max_size_in_memory)
self._in_eof = False
self._in_pos = 0
super(Buffer,self).__init__(fileobj,mode)
def _buffer_chunks(self):
chunk = self._buffer.read(16*1024)
if chunk == "":
yield chunk
else:
while chunk != "":
yield chunk
chunk = self._buffer.read(16*1024)
def _write_out_buffer(self):
if self._check_mode("r"):
self._read_rest()
if "a" in self.mode:
self._buffer.seek(self._in_pos)
self._fileobj.seek(self._in_pos)
else:
self._fileobj.seek(0)
self._buffer.seek(0)
else:
self._buffer.seek(0)
for chunk in self._buffer_chunks():
self._fileobj.write(chunk)
def flush(self):
# flush the buffer; we only write to the underlying file on close
self._buffer.flush()
def close(self):
if self.closed:
return
if self._check_mode("w"):
self._write_out_buffer()
super(Buffer,self).close()
self._buffer.close()
def _read(self,sizehint=-1):
# First return any data available from the buffer.
# Since we don't flush the buffer after every write, certain OSes
# (guess which!) will happy read junk data from the end of it.
# Instead, we explicitly read only up to self._in_pos.
if not self._in_eof:
buffered_size = self._in_pos - self._buffer.tell()
if sizehint >= 0:
buffered_size = min(sizehint,buffered_size)
else:
buffered_size = sizehint
data = self._buffer.read(buffered_size)
if data != "":
return data
# Then look for more data in the underlying file
if self._in_eof:
return None
data = self._fileobj.read(sizehint)
self._in_pos += len(data)
self._buffer.write(data)
if sizehint < 0 or len(data) < sizehint:
self._in_eof = True
self._buffer.flush()
return data
def _write(self,data,flushing=False):
self._buffer.write(data)
if self._check_mode("r") and not self._in_eof:
diff = self._buffer.tell() - self._in_pos
if diff > 0:
junk = self._fileobj.read(diff)
self._in_pos += len(junk)
if len(junk) < diff:
self._in_eof = True
self._buffer.flush()
def _seek(self,offset,whence):
# Ensure we've read enough to simply do the seek on the buffer
if self._check_mode("r") and not self._in_eof:
if whence == 0:
if offset > self._in_pos:
self._read_rest()
if whence == 1:
if self._buffer.tell() + offset > self._in_pos:
self._read_rest()
if whence == 2:
self._read_rest()
# Then just do it on the buffer...
self._buffer.seek(offset,whence)
def _tell(self):
return self._buffer.tell()
def _read_rest(self):
"""Read the rest of the input stream."""
if self._in_eof:
return
pos = self._buffer.tell()
self._buffer.seek(0,2)
data = self._fileobj.read(self._bufsize)
while data:
self._in_pos += len(data)
self._buffer.write(data)
data = self._fileobj.read(self._bufsize)
self._in_eof = True
self._buffer.flush()
self._buffer.seek(pos)
