def test_putfiles_and_list():
test_files = [('_testing/test_list/file_%d' % i, b'testing ... \n') for i in range(10)]
clusterIO.put_files(test_files, 'TES1')
listing = clusterIO.listdir('_testing/test_list/', 'TES1')
assert(len(listing) == 10)
def test_list_after_timeout():
test_files = [('_testing/test_list2/file_%d' % i, b'testing ... \n') for i in range(10)]
clusterIO.put_files(test_files, 'TES1')
time.sleep(2)
listing = clusterIO.listdirectory('_testing/test_list2/', 'TES1',timeout=.00001)
listing = clusterIO.listdirectory('_testing/test_list2/', 'TES1', timeout=5)
assert (len(listing) == 10)
def _queuePoll(self):
while self._dPoll:
try:
data = self._postQueue.get_nowait()
with self._lock:
self._numThreadsProcessing += 1
try:
files = []
for imNum, frame in data:
if self._aggregate_h5:
fn = '/'.join([
'__aggregate_h5', self.seriesName,
'frame%05d.pzf' % imNum
])
else:
fn = '/'.join(
[self.seriesName,
'frame%05d.pzf' % imNum])
pzf = PZFFormat.dumps(frame,
sequenceID=self.sequenceID,
frameNum=imNum,
**self.compSettings)
files.append((fn, pzf))
if len(files) > 0:
clusterIO.put_files(files,
serverfilter=self.clusterFilter)
except Exception as e:
self._last_thread_exception = e
logging.exception('Exception whilst putting files')
raise
finally:
with self._lock:
self._numThreadsProcessing -= 1
time.sleep(.01)
#print 't', len(data)
except Queue.Empty:
time.sleep(.01)
def test_putfiles_and_list():
test_files = [('_testing/test_list/file_%d' % i, 'testing ... \n')
for i in range(10000)]
t = time.time()
clusterIO.put_files(test_files[:1000], 'TEST')
print('putting 1000 small files took %3.5f s' % (time.time() - t))
t = time.time()
listing = clusterIO.listdir('_testing/test_list/')
print('Listing a directory with 1000 small files took %3.5f s' %
(time.time() - t))
t = time.time()
listing = clusterIO.listdir('_testing/test_list/')
print(
'Listing a directory with 1000 small files (from local cache) took %3.5f s'
% (time.time() - t))
time.sleep(5)
t = time.time()
listing = clusterIO.listdir('_testing/test_list/')
print(
'\nListing a directory with 1000 small files (second attempt after local cache expiration) took %3.5f s'
% (time.time() - t))
#print('Returned %d files\n' % len(listing))
#print(sorted([(f) for f in listing]))
assert (len(listing) == 1000)
#put remainder
t = time.time()
clusterIO.put_files(test_files[1000:], 'TEST')
print('putting 9000 small files took %3.5f s' % (time.time() - t))
time.sleep(2) #should be enough to invalidate local cache
t = time.time()
listing = clusterIO.listdir('_testing/test_list/')
print('Listing a directory with 10000 small files took %3.5f s' %
(time.time() - t))
assert (len(listing) == 10000)
t = time.time()
listing = clusterIO.listdir('_testing/test_list/')
print(
'Listing a directory with 10000 small files (from local cache) took %3.5f s'
% (time.time() - t))
assert (len(listing) == 10000)
time.sleep(2)
t = time.time()
listing = clusterIO.listdir('_testing/test_list/')
print(
'Listing a directory with 10000 small files (second attempt after local cache expiration) took %3.5f s'
% (time.time() - t))
time.sleep(2)
t = time.time()
listing = clusterIO.listdir('_testing/test_list/')
print(
'Listing a directory with 10000 small files (second attempt after local cache expiration) took %3.5f s'
% (time.time() - t))
assert (len(listing) == 10000)
#really hammer the server
print('\n\n stress testing with 100,000 files ...')
t = time.time()
for j in range(100):
clusterIO.put_files([('_testing/test_l2/series_%d/file_%d' %
(j, i), 'testing ... \n')
for i in range(1000)], 'TEST')
print('\nputting 100,000 small files in 100 directories took %3.5f s' %
(time.time() - t))
time.sleep(2)
t = time.time()
listing = clusterIO.listdir('_testing/test_l2/')
print(
'Listing a directory with 100 subdiectories, each with 1000 files took %3.5f s'
% (time.time() - t))
assert (len(listing) == 100)
time.sleep(2)
t = time.time()
listing = clusterIO.listdir('_testing/test_l2/')
print(
'Listing a directory with 100 subdirectories, each with 1000 files took %3.5f s, after cache expiry'
% (time.time() - t))
assert (len(listing) == 100)