def file_stream(path, chunk_size=512 * 1024):
with open(path, 'rb') as src:
wrapper = FileObjectThread(src, 'rb')
while True:
data = wrapper.read(chunk_size)
if not data:
return
yield data
def read(self, remote_path, mode='r+b', log_level=0):
# Download the file to a temporary location ..
with NamedTemporaryFile(mode,
suffix='zato-sftp-read.txt') as local_path:
self.download_file(remote_path, local_path.name)
# .. and read it in using a separate thread so as not to block the event loop.
thread_file = FileObjectThread(local_path)
data = thread_file.read()
thread_file.close()
return data
def from_snapshot(cls, file_path: str) -> 'Storage':
logger.info('Restoring from snapshot: file_path=%s', file_path)
try:
f = FileObjectThread(open('./tmp/data', 'rb'))
data = None
try:
data = f.read()
finally:
try:
f.close()
except:
pass
assert data.startswith(b'dredispy:1:')
storage = pickle.loads(data[11:])
except FileNotFoundError:
logger.info('Snapshot not found, creating empty storage')
storage = Storage()
storage.file_path = file_path
return storage