def test_merge_random():
items1 = sorted((random_name(4), random_name(8)) for _ in xrange(500))
items2 = sorted((random_name(4), random_name(8)) for _ in xrange(500))
x1 = sorted(dict(items1 + items2).items())
x2 = list(kv.merge_items(items1, items2))
assert x1 == x2
def __init__(self, tempstorage, buffersize=32 * 1024):
assert isinstance(buffersize, int)
self._tempstorage = tempstorage
self._tempname = "%s.ctmp" % random_name()
self._temp = tempstorage.create_file(self._tempname, mode="w+b")
self._buffersize = buffersize
self._streams = {}
def __init__(self, tempstorage, buffersize=32 * 1024):
assert isinstance(buffersize, int)
self._tempstorage = tempstorage
self._tempname = "%s.ctmp" % random_name()
self._temp = tempstorage.create_file(self._tempname, mode="w+b")
self._buffersize = buffersize
self._streams = {}
def __init__(self, basename="", parentdir=None, ext=".whoosh",
suppress=frozenset(), keepdir=False):
self.basename = basename or random_name(8)
self.parentdir = parentdir
dirname = parentdir or tempfile.mkdtemp(ext, self.basename)
self.dir = os.path.abspath(dirname)
self.suppress = suppress
self.keepdir = keepdir
def __init__(self, basename="", parentdir=None, ext=".whoosh",
suppress=frozenset(), keepdir=False):
self.basename = basename or random_name(8)
self.parentdir = parentdir
dirname = parentdir or tempfile.mkdtemp(ext, self.basename)
self.dir = os.path.abspath(dirname)
self.suppress = suppress
self.keepdir = keepdir
def _enqueue(self):
# Flush the documents stored in self.docbuffer to a file and put the
# filename on the job queue
docbuffer = self.docbuffer
dump = pickle.dump
length = len(docbuffer)
filename = "%s.doclist" % random_name()
with self.temp_storage().create_file(filename).raw_file() as f:
for item in docbuffer:
dump(item, f, -1)
if len(self.tasks) < self.procs:
self._new_task()
jobinfo = (filename, length)
self.jobqueue.put(jobinfo)
self.docbuffer = []
def _enqueue(self):
# Flush the documents stored in self.docbuffer to a file and put the
# filename on the job queue
docbuffer = self.docbuffer
dump = pickle.dump
length = len(docbuffer)
filename = "%s.doclist" % random_name()
with self.temp_storage().create_file(filename).raw_file() as f:
for item in docbuffer:
dump(item, f, -1)
if len(self.tasks) < self.procs:
self._new_task()
jobinfo = (filename, length)
self.jobqueue.put(jobinfo)
self.docbuffer = []
def temp_storage(self, name=None):
tdir = tempfile.gettempdir()
name = name or "%s.tmp" % random_name()
path = os.path.join(tdir, name)
tempstore = FileStorage(path)
return tempstore.create()
def temp_storage(self, name=None):
name = name or "%s.tmp" % random_name()
path = os.path.join(self.folder, name)
tempstore = FileStorage(path)
return tempstore.create()
def temp_storage(self, name=None):
tdir = tempfile.gettempdir()
name = name or "%s.tmp" % random_name()
path = os.path.join(tdir, name)
tempstore = FileStorage(path)
return tempstore.create()
def temp_storage(self, name=None):
name = name or "%s.tmp" % random_name()
path = os.path.join(self.folder, name)
tempstore = FileStorage(path)
return tempstore.create()
def _new_run(self):
path = "%s.run" % random_name()
f = self.tempstore.create_file(path).raw_file()
return path, f
def _random_id(cls, size=16):
return random_name(size=size)
def temp_storage(self, name=None):
name = name or "%s.tmp" % random_name()
path = os.path.join(self.folder, name)
return EncryptedFileStorage(path, self.masterkey).create()
def _random_id(cls, size=16):
return random_name(size=size)
def _new_run(self):
path = "%s.run" % random_name()
f = self.tempstore.create_file(path).raw_file()
return path, f
def temp_storage(self, name=None):
name = name or "%s.tmp" % random_name()
tempstore = RedisStore(self.redis, name)
return tempstore.create()
def temp_storage(self, name=None):
name = name or "%s.tmp" % random_name()
temp_store = SaeStorage(name)
return temp_store.create()