def _getNumFrames(self):
frameNames = [
f for f in clusterIO.listdir(self.sequenceName, self.clusterfilter)
if f.endswith('.pzf')
]
self.numFrames = len(frameNames)
self.lastShapeTime = time.time()
def test_dircache_purge():
testdata = b'foo bar\n'
for i in range(1050):
clusterIO.put_file('_testing/lots_of_folders/test_%d/test.txt' % i,
testdata, 'TEST')
listing = clusterIO.listdir('_testing/lots_of_folders/test_%d/' % i,
'TEST')
def test_putfiles_and_list():
test_files = [('_testing/test_list/file_%d' % i, b'testing ... \n') for i in range(10)]
clusterIO.put_files(test_files, 'TES1')
listing = clusterIO.listdir('_testing/test_list/', 'TES1')
assert(len(listing) == 10)
def test_putfiles_and_list():
test_files = [('_testing/test_list/file_%d' % i, 'testing ... \n')
for i in range(10000)]
t = time.time()
clusterIO.put_files(test_files[:1000], 'TEST')
print('putting 1000 small files took %3.5f s' % (time.time() - t))
t = time.time()
listing = clusterIO.listdir('_testing/test_list/')
print('Listing a directory with 1000 small files took %3.5f s' %
(time.time() - t))
t = time.time()
listing = clusterIO.listdir('_testing/test_list/')
print(
'Listing a directory with 1000 small files (from local cache) took %3.5f s'
% (time.time() - t))
time.sleep(5)
t = time.time()
listing = clusterIO.listdir('_testing/test_list/')
print(
'\nListing a directory with 1000 small files (second attempt after local cache expiration) took %3.5f s'
% (time.time() - t))
#print('Returned %d files\n' % len(listing))
#print(sorted([(f) for f in listing]))
assert (len(listing) == 1000)
#put remainder
t = time.time()
clusterIO.put_files(test_files[1000:], 'TEST')
print('putting 9000 small files took %3.5f s' % (time.time() - t))
time.sleep(2) #should be enough to invalidate local cache
t = time.time()
listing = clusterIO.listdir('_testing/test_list/')
print('Listing a directory with 10000 small files took %3.5f s' %
(time.time() - t))
assert (len(listing) == 10000)
t = time.time()
listing = clusterIO.listdir('_testing/test_list/')
print(
'Listing a directory with 10000 small files (from local cache) took %3.5f s'
% (time.time() - t))
assert (len(listing) == 10000)
time.sleep(2)
t = time.time()
listing = clusterIO.listdir('_testing/test_list/')
print(
'Listing a directory with 10000 small files (second attempt after local cache expiration) took %3.5f s'
% (time.time() - t))
time.sleep(2)
t = time.time()
listing = clusterIO.listdir('_testing/test_list/')
print(
'Listing a directory with 10000 small files (second attempt after local cache expiration) took %3.5f s'
% (time.time() - t))
assert (len(listing) == 10000)
#really hammer the server
print('\n\n stress testing with 100,000 files ...')
t = time.time()
for j in range(100):
clusterIO.put_files([('_testing/test_l2/series_%d/file_%d' %
(j, i), 'testing ... \n')
for i in range(1000)], 'TEST')
print('\nputting 100,000 small files in 100 directories took %3.5f s' %
(time.time() - t))
time.sleep(2)
t = time.time()
listing = clusterIO.listdir('_testing/test_l2/')
print(
'Listing a directory with 100 subdiectories, each with 1000 files took %3.5f s'
% (time.time() - t))
assert (len(listing) == 100)
time.sleep(2)
t = time.time()
listing = clusterIO.listdir('_testing/test_l2/')
print(
'Listing a directory with 100 subdirectories, each with 1000 files took %3.5f s, after cache expiry'
% (time.time() - t))
assert (len(listing) == 100)