def generate_thumbnail(self, content): content = file_from_content(content) uploaded_image = Image.open(content) if max(uploaded_image.size) >= self.max_size: uploaded_image.thumbnail((self.max_size, self.max_size), Image.BILINEAR) content = SpooledTemporaryFile(INMEMORY_FILESIZE) uploaded_image.save(content, uploaded_image.format) content.seek(0) thumbnail = uploaded_image.copy() thumbnail.thumbnail(self.thumbnail_size, Image.ANTIALIAS) thumbnail = thumbnail.convert('RGBA') thumbnail.format = self.thumbnail_format output = SpooledTemporaryFile(INMEMORY_FILESIZE) thumbnail.save(output, self.thumbnail_format) output.seek(0) thumb_path, thumb_id = self.store_content( output, 'thumb.%s' % self.thumbnail_format.lower()) self['thumb_id'] = thumb_id self['thumb_path'] = thumb_path thumbnail_file = self.thumb_file self['_thumb_public_url'] = thumbnail_file.public_url content.close()
def runTest(path,connectivity='Native',location='firefox'): #Note that timeout is a global maxiumum and includes queuing time! #it does not relate to how long the test runs #TODO Error Handling args = [ 'webpagetest', 'test', path, '--server', server, '--key', key, '--location', location, '--runs', '1', '--connectivity', connectivity, '--label',path, '--keepua', #Don't change the useragent to indicate this is a bot '--first', #Don't try for a repeat view '--poll','5' #How frequently to poll the web server for the result #,'--timeout',str(timeout) ] outT = SpooledTemporaryFile(mode='w+') #We can specify the size of the memory buffer here if we need. #Stops us hitting the buffer limit if use pipe. #cmd = "" #for arg in args: # cmd = cmd + arg + ' ' #print(cmd) result = run(args,stdout=outT,bufsize=4096,check=True) outT.seek(0) #Have to return to the start of the file to read it. result = outT.read() outT.close() output = loads(result) #String to JSON return output
def generate_thumbnail(self, content): content = file_from_content(content) try: uploaded_image = Image.open(content) except Exception: flask.abort(400, ValidationError( { 'message': 'File Format', 'object': {"error": "Format Incorrect"}, } )) if max(uploaded_image.size) >= self.max_size: uploaded_image.thumbnail((self.max_size, self.max_size), Image.BILINEAR) content = SpooledTemporaryFile(INMEMORY_FILESIZE) uploaded_image.save(content, uploaded_image.format) content.seek(0) thumbnail = uploaded_image.copy() thumbnail.thumbnail(self.thumbnail_size, Image.ANTIALIAS) thumbnail = thumbnail.convert('RGBA') thumbnail.format = self.thumbnail_format output = SpooledTemporaryFile(INMEMORY_FILESIZE) thumbnail.save(output, self.thumbnail_format) output.seek(0) thumb_path, thumb_id = self.store_content(output, 'thumb.%s' % self.thumbnail_format.lower()) self['thumb_id'] = thumb_id self['thumb_path'] = thumb_path thumbnail_file = self.thumb_file self['_thumb_public_url'] = thumbnail_file.public_url content.close()
def _do_execute_direct(self, code): shell = builtins.__xonsh_shell__ env = builtins.__xonsh_env__ out = io.StringIO() err = io.StringIO() enc = env.get('XONSH_ENCODING') out = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t', encoding=enc, newline='\n') err = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t', encoding=enc, newline='\n') try: with redirect_stdout(out), redirect_stderr(err), \ swap(builtins, '__xonsh_stdout_uncaptured__', out), \ swap(builtins, '__xonsh_stderr_uncaptured__', err), \ env.swap({'XONSH_STORE_STDOUT': False}): shell.default(code) interrupted = False except KeyboardInterrupt: interrupted = True output, error = '', '' if out.tell() > 0: out.seek(0) output = out.read() if err.tell() > 0: err.seek(0) error = err.read() out.close() err.close() return output, error, interrupted
class GoogleCloudFile(File): def __init__(self, name, mode, storage): self.name = name self.mime_type = mimetypes.guess_type(name)[0] self._mode = mode self._storage = storage # NOTE(mattrobenolt): This is the same change in behavior as in # the s3 backend. We're opting now to load the file # or metadata at this step. This means we won't actually # know a file doesn't exist until we try to read it. self.blob = FancyBlob(storage.download_url, self.name, storage.bucket) self._file = None self._is_dirty = False @property def size(self): return self.blob.size def _get_file(self): if self._file is None: with metrics.timer('filestore.read', instance='gcs'): self._file = SpooledTemporaryFile( max_size=self._storage.max_memory_size, suffix=".GSStorageFile", dir=None, ) if 'r' in self._mode: self._is_dirty = False self.blob.download_to_file(self._file) self._file.seek(0) return self._file def _set_file(self, value): self._file = value file = property(_get_file, _set_file) def read(self, num_bytes=None): if 'r' not in self._mode: raise AttributeError("File was not opened in read mode.") if num_bytes is None: num_bytes = -1 return super(GoogleCloudFile, self).read(num_bytes) def write(self, content): if 'w' not in self._mode: raise AttributeError("File was not opened in write mode.") self._is_dirty = True return super(GoogleCloudFile, self).write(force_bytes(content)) def close(self): if self._file is not None: if self._is_dirty: self.file.seek(0) self.blob.upload_from_file(self.file, content_type=self.mime_type) self._file.close() self._file = None
def close(self): """Send the change to the DFS, and close the file.""" self.flush() if 'c' not in self.mode: SpooledTemporaryFile.close(self)
class GoogleCloudFile(File): def __init__(self, name, mode, storage): self.name = name self.mime_type = mimetypes.guess_type(name)[0] self._mode = mode self._storage = storage self.blob = storage.bucket.get_blob(name) if not self.blob and 'w' in mode: self.blob = Blob(self.name, storage.bucket, chunk_size=storage.blob_chunk_size) self._file = None self._is_dirty = False @property def size(self): return self.blob.size def _get_file(self): if self._file is None: self._file = SpooledTemporaryFile( max_size=self._storage.max_memory_size, suffix=".GSStorageFile", dir=setting("FILE_UPLOAD_TEMP_DIR")) if 'r' in self._mode: self._is_dirty = False self.blob.download_to_file(self._file) self._file.seek(0) return self._file def _set_file(self, value): self._file = value file = property(_get_file, _set_file) def read(self, num_bytes=None): if 'r' not in self._mode: raise AttributeError("File was not opened in read mode.") if num_bytes is None: num_bytes = -1 return super().read(num_bytes) def write(self, content): if 'w' not in self._mode: raise AttributeError("File was not opened in write mode.") self._is_dirty = True return super().write(force_bytes(content)) def close(self): if self._file is not None: if self._is_dirty: self.blob.upload_from_file( self.file, rewind=True, content_type=self.mime_type, predefined_acl=self._storage.default_acl) self._file.close() self._file = None
def archive_to_repo(archive_path, repo, archive_type="tar"): """Downloads a archive from the specified path, extracts it into the repo's directory, commits any changes from the previous version and pushes it to github!!!!""" # Download the tarball and stick it in a tempfile r = requests.get(archive_path) tmp = SpooledTemporaryFile() tmp.write(r.content) tmp.seek(0) # Open the tempfile contents as an actual tarball if archive_type == "tar": archive = tarfile.open(fileobj=tmp) elif archive_type == "zip": archive = zipfile.open(tmp) else: raise ValueError("Unrecognized Archive Type") # Clear working files clear_working_dir(repo.working_dir) # Extract to the repo path archive.extract(repo.working_dir) # Add and commit everything! try: repo.git.add(".", A=True) repo.git.commit(m="New archive version") except: pass # May be that there was nothing new to commit # Cleanup, outta here! archive.close() tmp.close()
def submitTest(job,server,key): location = rowToLocation(job) print(location +' '+job['script']) args = [ 'webpagetest', 'test', job['script'], '--server', server, '--key', key, '--location', location, '--runs', '1', '--connectivity', 'Native', '--label',job['script'], '--keepua', #Don't change the useragent to indicate this is a bot '--first', #Don't try for a repeat view ] outT = SpooledTemporaryFile(mode='w+') #We can specify the size of the memory buffer here if we need. #Stops us hitting the buffer limit if use pipe. #cmd = "" #for arg in args: # cmd = cmd + arg + ' ' #print(cmd) result = run(args,stdout=outT,bufsize=4096,check=True) outT.seek(0) #Have to return to the start of the file to read it. result = outT.read() outT.close() output = loads(result) #String to JSON return output
def generate_thumbnail(self, content): content = file_from_content(content) uploaded_image = Image.open(content) if max(uploaded_image.size) >= self.max_size: uploaded_image.thumbnail((self.max_size, self.max_size), Image.BILINEAR) content = SpooledTemporaryFile(INMEMORY_FILESIZE) uploaded_image.save(content, uploaded_image.format) content.seek(0) thumbnail = uploaded_image.copy() thumbnail.thumbnail(self.thumbnail_size, Image.ANTIALIAS) thumbnail = thumbnail.convert('RGBA') thumbnail.format = self.thumbnail_format output = SpooledTemporaryFile(INMEMORY_FILESIZE) thumbnail.save(output, self.thumbnail_format) output.seek(0) thumb_path, thumb_id = self.store_content(output, 'thumb.%s' % self.thumbnail_format.lower()) self['thumb_id'] = thumb_id self['thumb_path'] = thumb_path thumbnail_file = self.thumb_file self['_thumb_public_url'] = thumbnail_file.public_url content.close()
def _save_content(self, obj, content, parameters): content.seek(0, os.SEEK_SET) content_autoclose = SpooledTemporaryFile() content_autoclose.write(content.read()) super(CustomS3Boto3Storage, self)._save_content(obj, content_autoclose, parameters) if not content_autoclose.closed: content_autoclose.close()
def ex(cmd): """Shell out a subprocess and return what it writes to stdout as a string""" in_mem_file = SpooledTemporaryFile(max_size=2048, mode="r+") subprocess.check_call(cmd, shell=True, stdout=in_mem_file) in_mem_file.seek(0) stdout = in_mem_file.read() in_mem_file.close() return stdout
def _shell_command(self, cmd): """Shell out a subprocess and return what it writes to stdout as a string""" in_mem_file = SpooledTemporaryFile(max_size=2048, mode="r+") check_call(cmd, shell=True, stdout=in_mem_file) in_mem_file.seek(0) stdout = in_mem_file.read() in_mem_file.close() return stdout
def image(type, spec=' ', ext='png'): # Parameters for `suml`. import suml.common import optparse options = optparse.Values(({ 'scruffy': True, 'png': ext == 'png', 'svg': ext == 'svg' or ext == 'pdf', 'font': os.getenv('SCRUFFY_FONT', suml.common.defaultScruffyFont()), 'shadow': False, })) from tempfile import SpooledTemporaryFile fout = SpooledTemporaryFile() # Execute Scruffy `suml`. if type == 'class': suml.yuml2dot.transform(spec, fout, options) elif type == 'sequence': suml.suml2pic.transform(spec, fout, options) else: return HTTPError(404, 'Unhandled diagram type.') # Retrieve the data generated. fout.seek(0) data = fout.read() fout.close() # Convert SVG to PDF? if ext == 'pdf': # Load SVG file. doc = xml.dom.expatbuilder.parseString(data) # Convert to a RLG drawing svg_renderer = svglib.svglib.SvgRenderer() svg_renderer.render(doc.documentElement) drawing = svg_renderer.finish() # Generate PDF. data = reportlab.graphics.renderPDF.drawToString(drawing) # Server the generated image. if ext == 'png': response.content_type = 'image/png' elif ext == 'svg': response.content_type = 'image/svg+xml' elif ext == 'pdf': response.content_type = 'application/pdf' else: return HTTPError(500, 'Unhandled extension type.') return data
class GoogleCloudFile(File): def __init__(self, name, mode, storage): self.name = name self.mime_type = mimetypes.guess_type(name)[0] self._mode = mode self._storage = storage self.blob = storage.bucket.get_blob(name) if not self.blob and 'w' in mode: self.blob = Blob(self.name, storage.bucket) self._file = None self._is_dirty = False @property def size(self): return self.blob.size def _get_file(self): if self._file is None: self._file = SpooledTemporaryFile( max_size=self._storage.max_memory_size, suffix=".GSStorageFile", dir=setting("FILE_UPLOAD_TEMP_DIR", None) ) if 'r' in self._mode: self._is_dirty = False self.blob.download_to_file(self._file) self._file.seek(0) return self._file def _set_file(self, value): self._file = value file = property(_get_file, _set_file) def read(self, num_bytes=None): if 'r' not in self._mode: raise AttributeError("File was not opened in read mode.") if num_bytes is None: num_bytes = -1 return super(GoogleCloudFile, self).read(num_bytes) def write(self, content): if 'w' not in self._mode: raise AttributeError("File was not opened in write mode.") self._is_dirty = True return super(GoogleCloudFile, self).write(force_bytes(content)) def close(self): if self._file is not None: if self._is_dirty: self.file.seek(0) self.blob.upload_from_file(self.file, content_type=self.mime_type) self._file.close() self._file = None
def close(self): """On close, seek to 0 and write the data via the API, then close() for realz """ logger.debug("close() called on WriteBuffer") self.seek(0) logger.debug("Attempting to create file at dir_path %s with name %s" % (self.path, self.filename)) self.fs.rc.fs.write_file(self, self.fullpath) SpooledTemporaryFile.close(self) # old-style class!
def getJSON(id, server): args = ['webpagetest', 'results', id, '--server', server] outT = SpooledTemporaryFile( mode='w+' ) #We can specify the size of the memory buffer here if we need. result = run(args, stdout=outT, bufsize=4096, check=True) outT.seek(0) #Have to return to the start of the file to read it. result = outT.read() outT.close() output = loads(result) #String to JSON return output
class GoogleCloudFile(File): def __init__(self, name, mode, storage, buffer_size=None): self.name = name self._mode = mode self._storage = storage self.blob = Blob(self.name, storage.bucket) self._file = None self._is_dirty = False @property def size(self): return self.blob.size def _get_file(self): if self._file is None: self._file = SpooledTemporaryFile( max_size=self._storage.max_memory_size, suffix=".GSStorageFile", dir=setting("FILE_UPLOAD_TEMP_DIR", None)) if 'r' in self._mode: self._is_dirty = False self.blob.download_to_file(self._file) self._file.seek(0) return self._file def _set_file(self, value): self._file = value file = property(_get_file, _set_file) def read(self, *args, **kwargs): if 'r' not in self._mode: raise AttributeError("File was not opened in read mode.") return super(GoogleCloudFile, self).read(*args, **kwargs) def write(self, content, *args, **kwargs): if 'w' not in self._mode: raise AttributeError("File was not opened in write mode.") self._is_dirty = True return super(GoogleCloudFile, self).write(force_bytes(content), *args, **kwargs) def close(self): if self._file is not None: if self._is_dirty: self.file.seek(0) content_type, _ = mimetypes.guess_type(self.name) content_type = getattr(self.file, 'content_type', content_type) size = getattr(self.file, 'size') self.blob.upload_from_file(self.file, content_type=content_type, size=size) self._file.close() self._file = None
class FTPHelper: """Base FTP helper class""" def __init__(self, ftp, name, mode): self.ftp = ftp self.name = name self.mode = mode self.file = SpooledTemporaryFile(64 * 1024) def close(self): """Close temporary file""" self.file.close()
class GCPBlob(File): def __init__( # noqa missed call to super self, file: Blob, name: Optional[str] = None, mode: Optional[str] = "rb") -> None: self.name = name or file.name self.mime_type = mimetypes.guess_type(self.name)[0] self.blob = file self._mode = mode self._file = None self._is_dirty = False @property def size(self) -> int: return self.blob.size def _get_file(self) -> SpooledTemporaryFile: if self._file is None: self._file = SpooledTemporaryFile(suffix=".GCPStorageFile", ) if 'r' in self._mode: self._is_dirty = False self.blob.download_to_file(self._file) self._file.seek(0) return self._file def _set_file(self, value): self._file = value file = property(_get_file, _set_file) def read(self, num_bytes=None): if 'r' not in self._mode: raise AttributeError("File was not opened in read mode.") if num_bytes is None: num_bytes = -1 return super().read(num_bytes) def write(self, content): if 'w' not in self._mode: raise AttributeError("File was not opened in write mode.") self._is_dirty = True return super().write(force_bytes(content)) def close(self) -> None: if self._file is not None: if self._is_dirty: self.blob.upload_from_file(self.file, rewind=True, content_type=self.mime_type) self._file.close() self._file = None
class RemoteFileBuffer(object): """File-like object providing buffer for local file operations. Instances of this class manage a local tempfile buffer corresponding to the contents of a remote file. All reads and writes happen locally, with the content being copied to the remote file only on flush() or close(). Instances of this class are returned by S3FS.open, but it is desgined to be usable by any FS subclass that manages remote files. """ def __init__(self,fs,path,mode): self.file = TempFile() self.fs = fs self.path = path self.mode = mode def __del__(self): if not self.closed: self.close() # This is lifted straight from the stdlib's tempfile.py def __getattr__(self,name): file = self.__dict__['file'] a = getattr(file, name) if not issubclass(type(a), type(0)): setattr(self, name, a) return a def __enter__(self): self.file.__enter__() return self def __exit__(self,exc,value,tb): self.close() return False def __iter__(self): return iter(self.file) def flush(self): self.file.flush() if "w" in self.mode or "a" in self.mode or "+" in self.mode: pos = self.file.tell() self.file.seek(0) self.fs.setcontents(self.path,self.file) self.file.seek(pos) def close(self): if "w" in self.mode or "a" in self.mode or "+" in self.mode: self.file.seek(0) self.fs.setcontents(self.path,self.file) self.file.close()
def upload_file(self, user, stream, expected_size, filename, force_coll_name=''): temp_file = None logger.debug('Upload Begin') logger.debug('Expected Size: ' + str(expected_size)) #is_anon = False size_rem = user.get_size_remaining() logger.debug('User Size Rem: ' + str(size_rem)) if size_rem < expected_size: return {'error': 'out_of_space'} if force_coll_name and not user.has_collection(force_coll_name): #if is_anon: # user.create_collection(force_coll, 'Temporary Collection') #else: #status = 'Collection {0} not found'.format(force_coll_name) return {'error': 'no_such_collection'} temp_file = SpooledTemporaryFile(max_size=BLOCK_SIZE) stream = CacheingLimitReader(stream, expected_size, temp_file) if filename.endswith('.har'): stream, expected_size = self.har2warc(filename, stream) temp_file.close() temp_file = stream infos = self.parse_uploaded(stream, expected_size) total_size = temp_file.tell() if total_size != expected_size: return { 'error': 'incomplete_upload', 'expected': expected_size, 'actual': total_size } upload_id, upload_key = self._init_upload_status(user, total_size, 1, filename=filename) return self.handle_upload(temp_file, upload_id, upload_key, infos, filename, user, force_coll_name, total_size)
def upload_file(self, user, stream, expected_size, filename, force_coll_name=''): """Upload WARC archive. :param User user: user :param stream: file object :param int expected_size: expected WARC archive size :param str filename: WARC archive filename :param str force_coll_name: name of collection to upload into :returns: upload information :rtype: dict """ temp_file = None logger.debug('Upload Begin') logger.debug('Expected Size: ' + str(expected_size)) #is_anon = False size_rem = user.get_size_remaining() logger.debug('User Size Rem: ' + str(size_rem)) if size_rem < expected_size: return {'error': 'out_of_space'} if force_coll_name and not user.has_collection(force_coll_name): #if is_anon: # user.create_collection(force_coll, 'Temporary Collection') #else: #status = 'Collection {0} not found'.format(force_coll_name) return {'error': 'no_such_collection'} temp_file = SpooledTemporaryFile(max_size=BLOCK_SIZE) stream = CacheingLimitReader(stream, expected_size, temp_file) if filename.endswith('.har'): stream, expected_size = self.har2warc(filename, stream) temp_file.close() temp_file = stream infos = self.parse_uploaded(stream, expected_size) total_size = temp_file.tell() if total_size != expected_size: return {'error': 'incomplete_upload', 'expected': expected_size, 'actual': total_size} upload_id, upload_key = self._init_upload_status(user, total_size, 1, filename=filename) return self.handle_upload(temp_file, upload_id, upload_key, infos, filename, user, force_coll_name, total_size)
def pipestring_process(cmd_string, stdin_string=''): """Pipe a python string to standard input for cmd_string >>> pipestring_process('grep 2', '1\\n2\\n3\\n') (0, '2\\n', '') """ f=SpooledTemporaryFile() f.write(stdin_string) f.seek(0) results=process(cmd_string, stdin=f) f.close() return results
def __init__(self, path, filename, fs, max_size=WRITE_BUFFER_SIZE): """We need the path so we can write the buffered file to the API""" SpooledTemporaryFile.__init__(self, max_size=max_size) # old-style! self.path = path self.filename = filename self.fs = fs self.fullpath = '' try: self.fullpath = self.create_file() except RequestError, e: SpooledTemporaryFile.close(self) raise FilesystemError(str(e))
def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False): """Execute user code.""" if len(code.strip()) == 0: return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], 'user_expressions': {}} env = builtins.__xonsh_env__ shell = builtins.__xonsh_shell__ hist = builtins.__xonsh_history__ enc = env.get('XONSH_ENCODING') out = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t', encoding=enc, newline='\n') err = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t', encoding=enc, newline='\n') try: with redirect_stdout(out), redirect_stderr(err), \ swap(builtins, '__xonsh_stdout_uncaptured__', out), \ swap(builtins, '__xonsh_stderr_uncaptured__', err), \ env.swap({'XONSH_STORE_STDOUT': False}): shell.default(code) interrupted = False except KeyboardInterrupt: interrupted = True if not silent: # stdout response if out.tell() > 0: out.seek(0) self._respond_in_chunks('stdout', out.read()) if err.tell() > 0: err.seek(0) self._respond_in_chunks('stderr', err.read()) if hasattr(builtins, '_') and builtins._ is not None: # rely on sys.displayhook functionality self._respond_in_chunks('stdout', pformat(builtins._)) builtins._ = None if len(hist) > 0 and out.tell() == 0 and err.tell() == 0: self._respond_in_chunks('stdout', hist.outs[-1]) out.close() err.close() if interrupted: return {'status': 'abort', 'execution_count': self.execution_count} rtn = 0 if len(hist) == 0 else hist.rtns[-1] if 0 < rtn: message = {'status': 'error', 'execution_count': self.execution_count, 'ename': '', 'evalue': str(rtn), 'traceback': []} else: message = {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], 'user_expressions': {}} return message
def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False): """Execute user code.""" if len(code.strip()) == 0: return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], 'user_expressions': {}} env = builtins.__xonsh_env__ shell = builtins.__xonsh_shell__ hist = builtins.__xonsh_history__ enc = env.get('XONSH_ENCODING') out = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t', encoding=enc, newline='\n') err = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t', encoding=enc, newline='\n') try: with redirect_stdout(out), redirect_stderr(err), \ swap(builtins, '__xonsh_stdout_uncaptured__', out), \ swap(builtins, '__xonsh_stderr_uncaptured__', err), \ env.swap({'XONSH_STORE_STDOUT': False}): shell.default(code) interrupted = False except KeyboardInterrupt: interrupted = True if not silent: # stdout response if out.tell() > 0: out.seek(0) self._respond_in_chunks('stdout', out.read()) if err.tell() > 0: err.seek(0) self._respond_in_chunks('stderr', err.read()) if hasattr(builtins, '_') and builtins._ is not None: # rely on sys.displayhook functionality self._respond_in_chunks('stdout', pformat(builtins._)) builtins._ = None if hist is not None and len(hist) > 0 and out.tell() == 0 and err.tell() == 0: self._respond_in_chunks('stdout', hist.outs[-1]) out.close() err.close() if interrupted: return {'status': 'abort', 'execution_count': self.execution_count} rtn = 0 if (hist is None or len(hist) == 0) else hist.rtns[-1] if 0 < rtn: message = {'status': 'error', 'execution_count': self.execution_count, 'ename': '', 'evalue': str(rtn), 'traceback': []} else: message = {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], 'user_expressions': {}} return message
class BloomStringSet: """ BloomSet implements set membership in a fixed memory footprint, but can reproduce the keys by streaming them to temporary files. As it's based on a bloom filter, there is a risk of false positives, which would cause some missing keyue, however the likelihood of such is controllable through the parameters. Temporary files are only created after they reach 5MB, otherwise stay in memory. cardinality: the estimated maximum number of unique elements. As one goes beyond this number, risk of collision increases, but sloooooowly. error_rate: the false positive rate you are comfortable with when the cardinality number is reached. """ def __init__(self, cardinality=10**6, error_rate=10**-9): self.bloom = BloomFilter(cardinality, error_rate) self.file = SpooledTemporaryFile(max_size=(2**20) * 5, mode='w') self.closed = False def add(self, key): if self.closed: raise Exception("Cannot add new element after attempting to read") if type(key) is not str: raise Exception("Can only use string keys for now") if key in self.bloom: return False self.bloom.add(key) self.file.write(key + "\n") def __contains__(self, key): return key in self.bloom def __iter__(self): self.closed = True self.file.seek(0) return self def __next__(self): line = self.file.readline() if line == '': self.file.close() raise StopIteration return line.strip() def __del__(self): self.file.close()
class TempInput(object): def __init__(self, inputstr): self.inputstr = inputstr def __enter__(self): self.tempfile = SpooledTemporaryFile() self.tempfile.write(self.inputstr) self.tempfile.seek(0) return self.tempfile def __exit__(self, type_, value, traceback): self.tempfile.close() return False
def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False): """Execute user code.""" if len(code.strip()) == 0: return {"status": "ok", "execution_count": self.execution_count, "payload": [], "user_expressions": {}} env = builtins.__xonsh_env__ shell = builtins.__xonsh_shell__ hist = builtins.__xonsh_history__ enc = env.get("XONSH_ENCODING") out = SpooledTemporaryFile(max_size=MAX_SIZE, mode="w+t", encoding=enc, newline="\n") err = SpooledTemporaryFile(max_size=MAX_SIZE, mode="w+t", encoding=enc, newline="\n") try: with redirect_stdout(out), redirect_stderr(err), swap(builtins, "__xonsh_stdout_uncaptured__", out), swap( builtins, "__xonsh_stderr_uncaptured__", err ), env.swap({"XONSH_STORE_STDOUT": False}): shell.default(code) interrupted = False except KeyboardInterrupt: interrupted = True if not silent: # stdout response if out.tell() > 0: out.seek(0) self._respond_in_chunks("stdout", out.read()) if err.tell() > 0: err.seek(0) self._respond_in_chunks("stderr", err.read()) if hasattr(builtins, "_") and builtins._ is not None: # rely on sys.displayhook functionality self._respond_in_chunks("stdout", pformat(builtins._)) builtins._ = None if len(hist) > 0 and out.tell() == 0 and err.tell() == 0: self._respond_in_chunks("stdout", hist.outs[-1]) out.close() err.close() if interrupted: return {"status": "abort", "execution_count": self.execution_count} rtn = 0 if len(hist) == 0 else hist.rtns[-1] if 0 < rtn: message = { "status": "error", "execution_count": self.execution_count, "ename": "", "evalue": str(rtn), "traceback": [], } else: message = {"status": "ok", "execution_count": self.execution_count, "payload": [], "user_expressions": {}} return message
def checkFinished(id, server): args = ['webpagetest', 'status', id, '--server', server] outT = SpooledTemporaryFile( mode='w+' ) #We can specify the size of the memory buffer here if we need. result = run(args, stdout=outT, bufsize=4096, check=True) outT.seek(0) #Have to return to the start of the file to read it. result = outT.read() outT.close() output = loads(result) #String to JSON if int(output['statusCode']) == 200: return True else: return False
def csv_fn(pydata): csvfile=SpooledTemporaryFile() csvw=csv.writer(csvfile) fieldnames=[] for h in pydata: for k in pydata[h]: if k not in fieldnames: fieldnames.append(k) if showheader: csvw.writerow(['system_name'] + fieldnames) for system_name in pydata: csvw.writerow([system_name] + [pydata[system_name].get(k, None) for k in fieldnames]) csvfile.seek(0) results=csvfile.read() csvfile.close() return results
def _save_content(self, obj, content, parameters): # Seek our content back to the start content.seek(0, os.SEEK_SET) # Create a temporary file that will write to disk after a specified size content_autoclose = SpooledTemporaryFile() # Write our original content into our copy that will be closed by boto3 content_autoclose.write(content.read()) # Upload the object which will auto close the content_autoclose instance super(S3DefaultStorage, self)._save_content(obj, content_autoclose, parameters) # Cleanup if this is fixed upstream our duplicate should always close if not content_autoclose.closed: content_autoclose.close()
class OrcCompressor(object): def __init__(self): self._buffer = SpooledTemporaryFile(mode='wb') def compress(self, data: bytes): self._buffer.write(data) def flush(self): if self._buffer: self._buffer.seek(0) df = pd.read_csv(self._buffer) with SpooledTemporaryFile(mode='wb') as wfd: df.to_parquet(wfd, engine='pyarrow') wfd.seek(0) return wfd.read() def __del__(self): self._buffer.close()
class InputStream(object): """ FCGI_STDIN or FCGI_DATA stream. Uses temporary file to store received data once max_mem bytes have been received. """ def __init__(self, max_mem=1024): self._file = SpooledTemporaryFile(max_mem) self._eof_received = Event() def __del__(self): self._file.close() def feed(self, data): if self._eof_received.is_set(): raise IOError('Feeding file beyond EOF mark') if not data: # EOF mark self._file.seek(0) self._eof_received.set() else: if isinstance(data, six.text_type): data = data.encode("ISO-8859-1") self._file.write(data) def __iter__(self): self._eof_received.wait() return iter(self._file) def read(self, size=-1): self._eof_received.wait() return self._file.read(size) def readline(self, size=-1): self._eof_received.wait() return self._file.readline(size) def readlines(self, sizehint=0): self._eof_received.wait() return self._file.readlines(sizehint) @property def eof_received(self): return self._eof_received.is_set()
class ParquetDecompressor(object): def __init__(self, sep=b','): self._buffer = SpooledTemporaryFile(mode='wb') self._result = SpooledTemporaryFile(mode='wb') self._sep = sep self.eof = False def decompress(self, data: bytes): self._buffer.write(data) return b'' def flush(self): self._buffer.seek(0) # TODO: to optimize it df = pd.read_parquet(self._buffer, engine='pyarrow') return df.to_csv(index=False).encode('utf-8') def __del__(self): self._buffer.close() self._result.close()
def _save_content(self, obj, content, parameters): """ We create a clone of the content file as when this is passed to boto3 it wrongly closes the file upon upload where as the storage backend expects it to still be open """ # Seek our content back to the start content.seek(0, os.SEEK_SET) # Create a temporary file that will write to disk after a specified size content_autoclose = SpooledTemporaryFile() # Write our original content into our copy that will be closed by boto3 content_autoclose.write(content.read()) # Upload the object which will auto close the content_autoclose instance super(CustomS3Boto3Storage, self)._save_content(obj, content_autoclose, parameters) # Cleanup if this is fixed upstream our duplicate should always close if not content_autoclose.closed: content_autoclose.close()
class TempInput(object): """ Acts as STDIN for a Popen process. Initialize with the string you want to send on startup. Enoding can be set with the optional 'encoding' arg. """ def __init__(self, inputstr, encoding=None): self.encoding = encoding or 'utf-8' self.inputstr = inputstr.encode(self.encoding) def __enter__(self): self.tempfile = SpooledTemporaryFile() self.tempfile.write(self.inputstr) self.tempfile.seek(0) return self.tempfile def __exit__(self, type_, value, traceback): self.tempfile.close() return False
def _save_content(self, obj, content, parameters): """ We create a clone of the content file as when this is passed to boto3 it wrongly closes the file upon upload where as the storage backend expects it to still be open """ # Seek our content back to the start content.seek(0, os.SEEK_SET) # Create a temporary file that will write to disk after a specified size content_autoclose = SpooledTemporaryFile() # Write our original content into our copy that will be closed by boto3 content_autoclose.write(content.read()) # Upload the object which will auto close the content_autoclose instance super(PrivateMediaStorage, self)._save_content(obj, content_autoclose, parameters) # Cleanup if this is fixed upstream our duplicate should always close if not content_autoclose.closed: content_autoclose.close()
def upload_to_s3(path, file, bucket_name=None): s3 = get_boto_client() bucket_name = bucket_name or current_app.config["AWS_S3_BUCKET"] # Need to clone the file because boto3 autocloses the file after uploading to s3, # preventing further use of the uploaded file. file_s3 = SpooledTemporaryFile() file_s3.write(file.read()) file_s3.seek(0) s3.upload_fileobj( file_s3, bucket_name, path, ExtraArgs={"ACL": "public-read", "ContentType": file.content_type}, ) if not file_s3.closed: file_s3.close() return bucket_name, path
def download_poster(image_url, out_dir): print(" [ Api ] Downloading cover art :") buffer = SpooledTemporaryFile(max_size=1e9) r = get(image_url, stream=True) if r.status_code == 200: downloaded = 0 filesize = int(r.headers['content-length']) for chunk in r.iter_content(chunk_size=1024): downloaded += len(chunk) buffer.write(chunk) print_progress_bar(downloaded, filesize) time.sleep(0.007) # print(downloaded/filesize) buffer.seek(0) print() if Path.joinpath(out_dir, 'image.jpg').exists(): os.remove(Path.joinpath(out_dir, 'image.jpg')) i = Image.open(BytesIO(buffer.read())) i.save(os.path.join(out_dir, 'image.jpg'), quality=100) buffer.close()
class LLDB: def _parseStackTrace(self, jibberish): not_jibberish = re.findall(r'\(lldb\) bt(.*)\(lldb\)', jibberish, re.DOTALL) if len(not_jibberish) != 0: return not_jibberish[0].replace(' frame ', '') else: return 'Stack Trace failed:', jibberish def _waitForResponse(self, wait=True): while wait: self.lldb_stdout.seek(self.last_position) for line in self.lldb_stdout: if line == '(lldb) ': self.last_position = self.lldb_stdout.tell() return True time.sleep(0.05) time.sleep(0.05) return True def getStackTrace(self, pid): lldb_commands = [ 'attach -p ' + pid + '\n', 'bt\n', 'quit\n', 'Y\n' ] self.lldb_stdout = SpooledTemporaryFile() self.last_position = 0 lldb_process = subprocess.Popen(['lldb', '-x'], stdin=subprocess.PIPE, stdout=self.lldb_stdout, stderr=self.lldb_stdout) while lldb_process.poll() == None: for command in lldb_commands: if command == lldb_commands[-1]: lldb_commands = [] if self._waitForResponse(False): # I have seen LLDB exit out from under us try: lldb_process.stdin.write(command) except: pass elif self._waitForResponse(): lldb_process.stdin.write(command) self.lldb_stdout.seek(0) stack_trace = self._parseStackTrace(self.lldb_stdout.read()) self.lldb_stdout.close() return stack_trace
class GDB: def _parseStackTrace(self, jibberish): not_jibberish = re.findall(r'\(gdb\) (#.*)\(gdb\)', jibberish, re.DOTALL) if len(not_jibberish) != 0: return not_jibberish[0] else: return 'Stack Trace failed:', jibberish def _waitForResponse(self, wait=True): while wait: self.gdb_stdout.seek(self.last_position) for line in self.gdb_stdout: if line == '(gdb) ': self.last_position = self.gdb_stdout.tell() return True time.sleep(0.05) time.sleep(0.05) return True def getStackTrace(self, pid): gdb_commands = [ 'attach ' + pid + '\n', 'set verbose off\n', 'thread\n', 'apply\n', 'all\n', 'bt\n', 'quit\n', 'y\n' ] self.gdb_stdout = SpooledTemporaryFile() self.last_position = 0 gdb_process = subprocess.Popen(['gdb', '-nx'], stdin=subprocess.PIPE, stdout=self.gdb_stdout, stderr=self.gdb_stdout) while gdb_process.poll() == None: for command in gdb_commands: if command == gdb_commands[-1]: gdb_commands = [] elif self._waitForResponse(): # I have seen GDB exit out from under us try: gdb_process.stdin.write(command) except: pass self.gdb_stdout.seek(0) stack_trace = self._parseStackTrace(self.gdb_stdout.read()) self.gdb_stdout.close() return stack_trace
class StartFinish(object): def __init__(self): pass def start(self, args): self.outFile = SpooledTemporaryFile() self.errFile = SpooledTemporaryFile() self.cmdline = list2cmdline(args) print 'starting: ' + self.cmdline self.process = Popen(args, stderr=self.errFile, stdout=self.outFile, universal_newlines=False) self.process_start = time() def finish(self, timeout): print 'finishing "' + self.cmdline + '"' kill_time = self.process_start + timeout self.process_end = time() while self.process.poll() is None: self.process_end = time() if self.process_end < kill_time: sleep(0.1) else: self.process.kill() raise Exception('timeout "' + self.cmdline + '"') # read temp streams from start self.outFile.seek(0) self.errFile.seek(0) self.out = self.outFile.read() self.err = self.errFile.read() self.outFile.close() self.errFile.close() print 'out:', self.out print 'err:', self.err print 'returncode:', self.process.returncode print 'elapsed:', self.process_end - self.process_start return self.process.returncode
class Buffer(FileWrapper): """Class implementing buffering of input and output streams. This class uses a separate buffer file to hold the contents of the underlying file while they are being manipulated. As data is read it is duplicated into the buffer, and data is written from the buffer back to the file on close. """ def __init__(self, fileobj, mode=None, max_size_in_memory=1024 * 8): """Buffered file wrapper constructor.""" self._buffer = SpooledTemporaryFile(max_size=max_size_in_memory) self._in_eof = False self._in_pos = 0 self._was_truncated = False super(Buffer, self).__init__(fileobj, mode) def _buffer_size(self): try: return len(self._buffer.file.getvalue()) except AttributeError: return os.fstat(self._buffer.fileno()).st_size def _buffer_chunks(self): chunk = self._buffer.read(16 * 1024) if chunk == "": yield chunk else: while chunk != "": yield chunk chunk = self._buffer.read(16 * 1024) def _write_out_buffer(self): if self._check_mode("r"): self._read_rest() if "a" in self.mode: self._buffer.seek(self._in_pos) self._fileobj.seek(self._in_pos) else: self._fileobj.seek(0) self._buffer.seek(0) else: self._buffer.seek(0) if self._was_truncated: self._fileobj.truncate(0) self._was_truncated = False for chunk in self._buffer_chunks(): self._fileobj.write(chunk) def flush(self): # flush the buffer; we only write to the underlying file on close self._buffer.flush() def close(self): if self.closed: return if self._check_mode("w"): self._write_out_buffer() super(Buffer, self).close() self._buffer.close() def _read(self, sizehint=-1): # First return any data available from the buffer. # Since we don't flush the buffer after every write, certain OSes # (guess which!) will happily read junk data from the end of it. # Instead, we explicitly read only up to self._in_pos. if not self._in_eof: buffered_size = self._in_pos - self._buffer.tell() if sizehint >= 0: buffered_size = min(sizehint, buffered_size) else: buffered_size = sizehint data = self._buffer.read(buffered_size) if data != "": return data # Then look for more data in the underlying file if self._in_eof: return None data = self._fileobj.read(sizehint) self._in_pos += len(data) self._buffer.write(data) if sizehint < 0 or len(data) < sizehint: self._in_eof = True self._buffer.flush() return data def _write(self, data, flushing=False): self._buffer.write(data) if self._check_mode("r") and not self._in_eof: diff = self._buffer.tell() - self._in_pos if diff > 0: junk = self._fileobj.read(diff) self._in_pos += len(junk) if len(junk) < diff: self._in_eof = True self._buffer.flush() def _seek(self, offset, whence): # Ensure we've read enough to simply do the seek on the buffer if self._check_mode("r") and not self._in_eof: if whence == 0: if offset > self._in_pos: self._read_rest() if whence == 1: if self._buffer.tell() + offset > self._in_pos: self._read_rest() if whence == 2: self._read_rest() # Then just do it on the buffer... self._buffer.seek(offset, whence) def _tell(self): return self._buffer.tell() def _truncate(self, size): if self._check_mode("r") and not self._in_eof: if size > self._in_pos: self._read_rest() self._in_eof = True try: self._buffer.truncate(size) except TypeError: et, ev, tb = sys.exc_info() # SpooledTemporaryFile.truncate() doesn't accept size paramter. try: self._buffer._file.truncate(size) except Exception: raise et, ev, tb # StringIO objects don't truncate to larger size correctly. if hasattr(self._buffer, "_file"): _file = self._buffer._file if hasattr(_file, "getvalue"): if len(_file.getvalue()) != size: curpos = _file.tell() _file.seek(0, 2) _file.write("\x00" * (size - len(_file.getvalue()))) _file.seek(curpos) self._was_truncated = True def _read_rest(self): """Read the rest of the input stream.""" if self._in_eof: return pos = self._buffer.tell() self._buffer.seek(0, 2) data = self._fileobj.read(self._bufsize) while data: self._in_pos += len(data) self._buffer.write(data) data = self._fileobj.read(self._bufsize) self._in_eof = True self._buffer.flush() self._buffer.seek(pos)
class IncrementalWriter: """A streaming file writer for point clouds. Using the IncrementalWriter with spooled temporary files, which are only flushed to disk if they go above the given size, allows for streaming points to disk even when the header is unknown in advance. This allows some nice tricks, including splitting a point cloud into multiple files in a single pass, without memory issues. """ # pylint:disable=too-few-public-methods def __init__(self, filename: str, header: PlyHeader, utm: UTM_Coord=None, buffer=2**22) -> None: """ Args: filename: final place to save the file on disk. source_fname: source file for the pointcloud; used to detect file format for metadata etc. buffer (int): The number of bytes to hold in RAM before flushing the temporary file to disk. Default 1MB, which holds ~8300 points - enough for most objects but still practical to hold thousands in memory. Set a smaller buffer for large forests. """ self.filename = filename self.temp_storage = SpooledTemporaryFile(max_size=buffer, mode='w+b') self.count = 0 self.utm = utm self.header = header # Always write in big-endian mode; only store type information self.binary = struct.Struct('>' + header.form_str[1:]) def __call__(self, point) -> None: """Add a single point to this pointcloud, saving in binary format. Args: point (namedtuple): vertex attributes for the point, eg xyzrgba. """ self.temp_storage.write(self.binary.pack(*point)) self.count += 1 def __del__(self): """Flush data to disk and clean up.""" to_ply_types = {v: k for k, v in PLY_TYPES.items()} properties = ['property {t} {n}'.format(t=t, n=n) for t, n in zip( (to_ply_types[p] for p in self.header.form_str[1:]), self.header.names)] head = ['ply', 'format binary_big_endian 1.0', 'element vertex {}'.format(self.count), '\n'.join(properties), 'end_header'] if self.utm is not None: head.insert(-1, 'comment UTM x y zone north ' + '{0.x} {0.y} {0.zone} {0.north}'.format(self.utm)) if not os.path.isdir(os.path.dirname(self.filename)): os.makedirs(os.path.dirname(self.filename)) with open(self.filename, 'wb') as f: f.write(('\n'.join(head) + '\n').encode('ascii')) self.temp_storage.seek(0) chunk = self.temp_storage.read(8192) while chunk: f.write(chunk) chunk = self.temp_storage.read(8192) self.temp_storage.close()
def runTest(test, testnum): # test is a tuple of ( filename , usedb<bool> ) # filename should be a js file to run # usedb is true if the test expects a mongod to be running (path, usedb) = test (ignore, ext) = os.path.splitext(path) # the dbtests know how to format themselves nicely, we'll detect if we're running them and if # so, we won't mess with the output is_test_binary = False if skipTest(path): if quiet: sys.stdout.write("skip %d %s\n" % (testnum, os.path.basename(path))) sys.stdout.flush() else: print "skipping " + path return if file_of_commands_mode: # smoke.py was invoked like "--mode files --from-file foo", # so don't try to interpret the test path too much if os.sys.platform == "win32": argv = [path] else: argv = shlex.split(path) path = argv[0] # if the command is a python script, use the script name if os.path.basename(path) in ('python', 'python.exe'): path = argv[1] elif ext == ".js": argv = [shell_executable, "--port", mongod_port, '--authenticationMechanism', authMechanism] if not usedb: argv += ["--nodb"] if small_oplog or small_oplog_rs: argv += ["--eval", 'testingReplication = true;'] if use_ssl: argv += ["--ssl", "--sslPEMKeyFile", "jstests/libs/client.pem", "--sslCAFile", "jstests/libs/ca.pem"] argv += [path] elif ext in ["", ".exe"]: # Blech. if os.path.basename(path) in ["test", "test.exe", "perftest", "perftest.exe"]: argv = [path] if os.path.basename(path) in ["test", "test.exe"]: is_test_binary = True # more blech elif os.path.basename(path) in ['mongos', 'mongos.exe']: argv = [path, "--test"] else: argv = [test_path and os.path.abspath(os.path.join(test_path, path)) or path, "--port", mongod_port] else: raise Bug("fell off in extension case: %s" % path) if keyFile: f = open(keyFile, 'r') keyFileData = re.sub(r'\s', '', f.read()) # Remove all whitespace f.close() os.chmod(keyFile, stat.S_IRUSR | stat.S_IWUSR) else: keyFileData = None mongo_test_filename = os.path.basename(path) if 'sharedclient' in path: mongo_test_filename += "-sharedclient" # sys.stdout.write() is more atomic than print, so using it prevents # lines being interrupted by, e.g., child processes if quiet and not is_test_binary: vlog = tests_log qlog = sys.stdout tlog = sys.stderr else: vlog = sys.stdout qlog = None tlog = None vlog.write(" *******************************************\n") vlog.write(" Test : %s ...\n" % mongo_test_filename) vlog.flush() # FIXME: we don't handle the case where the subprocess # hangs... that's bad. if ( argv[0].endswith( 'mongo' ) or argv[0].endswith( 'mongo.exe' ) ) and not '--eval' in argv : evalString = 'TestData = new Object();' + \ 'TestData.testPath = "' + path + '";' + \ 'TestData.testFile = "' + os.path.basename( path ) + '";' + \ 'TestData.testName = "' + re.sub( ".js$", "", os.path.basename( path ) ) + '";' + \ 'TestData.noJournal = ' + ternary( no_journal ) + ";" + \ 'TestData.noJournalPrealloc = ' + ternary( no_preallocj ) + ";" + \ 'TestData.auth = ' + ternary( auth ) + ";" + \ 'TestData.keyFile = ' + ternary( keyFile , '"' + str(keyFile) + '"' , 'null' ) + ";" + \ 'TestData.keyFileData = ' + ternary( keyFile , '"' + str(keyFileData) + '"' , 'null' ) + ";" if os.sys.platform == "win32": # double quotes in the evalString on windows; this # prevents the backslashes from being removed when # the shell (i.e. bash) evaluates this string. yuck. evalString = evalString.replace('\\', '\\\\') if auth and usedb: evalString += 'jsTest.authenticate(db.getMongo());' argv = argv + [ '--eval', evalString] if argv[0].endswith( 'test' ) and no_preallocj : argv = argv + [ '--nopreallocj' ] vlog.write(" Command : %s\n" % ' '.join(argv)) vlog.write(" Date : %s\n" % datetime.now().ctime()) vlog.flush() tempfile = SpooledTemporaryFile(max_size=16*1024*1024) try: os.environ['MONGO_TEST_FILENAME'] = mongo_test_filename t1 = time.time() r = call(buildlogger(argv), cwd=test_path, # the dbtests know how to format their own output nicely stdout=ternary(is_test_binary, vlog, tempfile)) t2 = time.time() del os.environ['MONGO_TEST_FILENAME'] vlog.write(" %fms\n" % ((t2 - t1) * 1000)) vlog.flush() if not is_test_binary: tempfile.seek(0) for line in tempfile: vlog.write(line) vlog.flush() if quiet: if r == 0: qlog.write('ok %d %s\n' % (testnum, os.path.basename(path))) else: qlog.write('not ok %d %s # exit %d\n' % (testnum, os.path.basename(path), r)) qlog.flush() if r != 0: tempfile.seek(0) for line in tempfile: tlog.write(line) tlog.flush() if r != 0: raise TestExitFailure(path, r) finally: tempfile.close() if r != 0: raise TestExitFailure(path, r) if start_mongod: try: c = Connection(host="127.0.0.1", port=int(mongod_port), ssl=use_ssl) except Exception,e: print "Exception from pymongo: ", e raise TestServerFailure(path)
def runTest(test): # test is a tuple of ( filename , usedb<bool> ) # filename should be a js file to run # usedb is true if the test expects a mongod to be running (path, usedb) = test (ignore, ext) = os.path.splitext(path) if skipTest(path): print "skipping " + path return if file_of_commands_mode: # smoke.py was invoked like "--mode files --from-file foo", # so don't try to interpret the test path too much if os.sys.platform == "win32": argv = [path] else: argv = shlex.split(path) path = argv[0] # if the command is a python script, use the script name if os.path.basename(path) in ('python', 'python.exe'): path = argv[1] elif ext == ".js": argv = [shell_executable, "--port", mongod_port] if not usedb: argv += ["--nodb"] if small_oplog or small_oplog_rs: argv += ["--eval", 'testingReplication = true;'] argv += [path] elif ext in ["", ".exe"]: # Blech. if os.path.basename(path) in ["test", "test.exe", "perftest", "perftest.exe"]: argv = [path] # more blech elif os.path.basename(path) in ['mongos', 'mongos.exe']: argv = [path, "--test"] else: argv = [test_path and os.path.abspath(os.path.join(test_path, path)) or path, "--port", mongod_port] else: raise Bug("fell off in extenstion case: %s" % path) if keyFile: f = open(keyFile, 'r') keyFileData = re.sub(r'\s', '', f.read()) # Remove all whitespace f.close() else: keyFileData = None # sys.stdout.write() is more atomic than print, so using it prevents # lines being interrupted by, e.g., child processes if quiet: vlog = tests_log qlog = sys.stdout else: vlog = sys.stdout qlog = None vlog.write(" *******************************************\n") vlog.write(" Test : %s ...\n" % os.path.basename(path)) vlog.flush() # FIXME: we don't handle the case where the subprocess # hangs... that's bad. if ( argv[0].endswith( 'mongo' ) or argv[0].endswith( 'mongo.exe' ) ) and not '--eval' in argv : evalString = 'TestData = new Object();' + \ 'TestData.testPath = "' + path + '";' + \ 'TestData.testFile = "' + os.path.basename( path ) + '";' + \ 'TestData.testName = "' + re.sub( ".js$", "", os.path.basename( path ) ) + '";' + \ 'TestData.noJournal = ' + ternary( no_journal ) + ";" + \ 'TestData.noJournalPrealloc = ' + ternary( no_preallocj ) + ";" + \ 'TestData.auth = ' + ternary( auth ) + ";" + \ 'TestData.keyFile = ' + ternary( keyFile , '"' + str(keyFile) + '"' , 'null' ) + ";" + \ 'TestData.keyFileData = ' + ternary( keyFile , '"' + str(keyFileData) + '"' , 'null' ) + ";" if os.sys.platform == "win32": # double quotes in the evalString on windows; this # prevents the backslashes from being removed when # the shell (i.e. bash) evaluates this string. yuck. evalString = evalString.replace('\\', '\\\\') if auth and usedb: evalString += 'jsTest.authenticate(db.getMongo());' argv = argv + [ '--eval', evalString] if argv[0].endswith( 'test' ) and no_preallocj : argv = argv + [ '--nopreallocj' ] vlog.write(" Command : %s\n" % ' '.join(argv)) vlog.write(" Date : %s\n" % datetime.now().ctime()) vlog.flush() tempfile = SpooledTemporaryFile(max_size=16*1024*1024) try: os.environ['MONGO_TEST_FILENAME'] = os.path.basename(path) t1 = time.time() r = call(buildlogger(argv), cwd=test_path, stdout=tempfile) t2 = time.time() del os.environ['MONGO_TEST_FILENAME'] vlog.write(" %fms\n" % ((t2 - t1) * 1000)) vlog.flush() tempfile.seek(0) for line in tempfile: vlog.write(line) vlog.flush() if quiet: qlog.write("%s%s : %s\n" % ((" " * max(0, 64 - len(os.path.basename(path)))), os.path.basename(path), ternary(r == 0, "ok", "fail"))) if r != 0: tempfile.seek(0) for line in tempfile: qlog.write(line) qlog.flush() if r != 0: raise TestExitFailure(path, r) finally: tempfile.close()
class Buffer(FileWrapper): """Class implementing buffereing of input and output streams. This class uses a separate buffer file to hold the contents of the underlying file while they are being manipulated. As data is read it is duplicated into the buffer, and data is written from the buffer back to the file on close. """ def __init__(self,fileobj,mode=None,max_size_in_memory=1024*8): """Buffered file wrapper constructor.""" self._buffer = SpooledTemporaryFile(max_size=max_size_in_memory) self._in_eof = False self._in_pos = 0 super(Buffer,self).__init__(fileobj,mode) def _buffer_chunks(self): chunk = self._buffer.read(16*1024) if chunk == "": yield chunk else: while chunk != "": yield chunk chunk = self._buffer.read(16*1024) def _write_out_buffer(self): if self._check_mode("r"): self._read_rest() if "a" in self.mode: self._buffer.seek(self._in_pos) self._fileobj.seek(self._in_pos) else: self._fileobj.seek(0) self._buffer.seek(0) else: self._buffer.seek(0) for chunk in self._buffer_chunks(): self._fileobj.write(chunk) def flush(self): # flush the buffer; we only write to the underlying file on close self._buffer.flush() def close(self): if self.closed: return if self._check_mode("w"): self._write_out_buffer() super(Buffer,self).close() self._buffer.close() def _read(self,sizehint=-1): # First return any data available from the buffer. # Since we don't flush the buffer after every write, certain OSes # (guess which!) will happy read junk data from the end of it. # Instead, we explicitly read only up to self._in_pos. if not self._in_eof: buffered_size = self._in_pos - self._buffer.tell() if sizehint >= 0: buffered_size = min(sizehint,buffered_size) else: buffered_size = sizehint data = self._buffer.read(buffered_size) if data != "": return data # Then look for more data in the underlying file if self._in_eof: return None data = self._fileobj.read(sizehint) self._in_pos += len(data) self._buffer.write(data) if sizehint < 0 or len(data) < sizehint: self._in_eof = True self._buffer.flush() return data def _write(self,data,flushing=False): self._buffer.write(data) if self._check_mode("r") and not self._in_eof: diff = self._buffer.tell() - self._in_pos if diff > 0: junk = self._fileobj.read(diff) self._in_pos += len(junk) if len(junk) < diff: self._in_eof = True self._buffer.flush() def _seek(self,offset,whence): # Ensure we've read enough to simply do the seek on the buffer if self._check_mode("r") and not self._in_eof: if whence == 0: if offset > self._in_pos: self._read_rest() if whence == 1: if self._buffer.tell() + offset > self._in_pos: self._read_rest() if whence == 2: self._read_rest() # Then just do it on the buffer... self._buffer.seek(offset,whence) def _tell(self): return self._buffer.tell() def _read_rest(self): """Read the rest of the input stream.""" if self._in_eof: return pos = self._buffer.tell() self._buffer.seek(0,2) data = self._fileobj.read(self._bufsize) while data: self._in_pos += len(data) self._buffer.write(data) data = self._fileobj.read(self._bufsize) self._in_eof = True self._buffer.flush() self._buffer.seek(pos)