def test_issue_241(self):
inode = self.inode
# Create block
with self.cache.get(inode, 0) as fh:
fh.write(self.random_data(500))
# "Fill" cache
self.cache.cache.max_entries = 0
# Mock locking to reproduce race condition
mlock = MockMultiLock(self.cache.mlock)
with patch.object(self.cache, 'mlock', mlock):
# Start first expiration run, will block in upload
thread1 = AsyncFn(self.cache.expire)
thread1.start()
# Remove the object while the expiration thread waits
# for it to become available.
thread2 = AsyncFn(self.cache.remove, inode, 0, 1)
thread2.start()
mlock.yield_to(thread2)
thread2.join_and_raise(timeout=10)
assert not thread2.is_alive()
# Create a new object for the same block
with self.cache.get(inode, 0) as fh:
fh.write(self.random_data(500))
# Continue first expiration run
mlock.yield_to(thread1, block=False)
thread1.join_and_raise(timeout=10)
assert not thread1.is_alive()
def main(args=None):
options = parse_args(args)
src_backend_factory = options.src_backend_factory
dst_backend_factory = options.dst_backend_factory
# Try to access both backends before starting threads
try:
src_backend_factory().lookup('s3ql_metadata')
try:
dst_backend_factory().lookup('some random object')
except NoSuchObject:
pass
except DanglingStorageURLError as exc:
raise QuietError(str(exc)) from None
queue = Queue(maxsize=options.threads)
threads = []
for _ in range(options.threads):
t = AsyncFn(copy_loop, queue, src_backend_factory,
dst_backend_factory)
# Don't wait for worker threads, gives deadlock if main thread
# terminates with exception
t.daemon = True
t.start()
threads.append(t)
with src_backend_factory() as backend:
stamp1 = 0
for (i, key) in enumerate(backend):
stamp2 = time.time()
if stamp2 - stamp1 > 1:
stamp1 = stamp2
sys.stdout.write('\rCopied %d objects so far...' % i)
sys.stdout.flush()
# Terminate early if any thread failed with an exception
for t in threads:
if not t.is_alive():
t.join_and_raise()
# Avoid blocking if all threads terminated
while True:
try:
queue.put(key, timeout=1)
except QueueFull:
pass
else:
break
for t in threads:
if not t.is_alive():
t.join_and_raise()
sys.stdout.write('\n')
queue.maxsize += len(threads)
for t in threads:
queue.put(None)
for t in threads:
t.join_and_raise()
def main(args=None):
options = parse_args(args)
src_backend_factory = options.src_backend_factory
dst_backend_factory = options.dst_backend_factory
# Try to access both backends before starting threads
try:
src_backend_factory().lookup('s3ql_metadata')
try:
dst_backend_factory().lookup('some random object')
except NoSuchObject:
pass
except DanglingStorageURLError as exc:
raise QuietError(str(exc)) from None
queue = Queue(maxsize=options.threads)
threads = []
for _ in range(options.threads):
t = AsyncFn(copy_loop, queue, src_backend_factory, dst_backend_factory)
# Don't wait for worker threads, gives deadlock if main thread
# terminates with exception
t.daemon = True
t.start()
threads.append(t)
with src_backend_factory() as backend:
stamp1 = 0
for (i, key) in enumerate(backend):
stamp2 = time.time()
if stamp2 - stamp1 > 1:
stamp1 = stamp2
sys.stdout.write('\rCopied %d objects so far...' % i)
sys.stdout.flush()
# Terminate early if any thread failed with an exception
for t in threads:
if not t.is_alive():
t.join_and_raise()
# Avoid blocking if all threads terminated
while True:
try:
queue.put(key, timeout=1)
except QueueFull:
pass
else:
break
for t in threads:
if not t.is_alive():
t.join_and_raise()
sys.stdout.write('\n')
queue.maxsize += len(threads)
for t in threads:
queue.put(None)
for t in threads:
t.join_and_raise()
def main(args=None):
if args is None:
args = sys.argv[1:]
options = parse_args(args)
setup_logging(options)
try:
options.storage_url = options.src_storage_url
src_backend_factory = get_backend_factory(options, plain=True)
options.storage_url = options.dst_storage_url
dst_backend_factory = get_backend_factory(options, plain=True)
except DanglingStorageURLError as exc:
raise QuietError(str(exc)) from None
src_backends = [ src_backend_factory() for _ in range(options.threads) ]
dst_backends = [ dst_backend_factory() for _ in range(options.threads) ]
queue = Queue(maxsize=options.threads)
threads = []
for (src_backend, dst_backend) in zip(src_backends, dst_backends):
t = AsyncFn(copy_loop, queue, src_backend, dst_backend)
# Don't wait for worker threads, gives deadlock if main thread
# terminates with exception
t.daemon = True
t.start()
threads.append(t)
stamp1 = 0
for (i, key) in enumerate(src_backends[-1]):
stamp2 = time.time()
if stamp2 - stamp1 > 1:
stamp1 = stamp2
sys.stdout.write('\rCopied %d objects so far...' % i)
sys.stdout.flush()
# Terminate early if any thread failed with an exception
for t in threads:
if not t.is_alive():
t.join_and_raise()
# Avoid blocking if all threads terminated
while True:
try:
queue.put(key, timeout=1)
except QueueFull:
pass
else:
break
for t in threads:
if not t.is_alive():
t.join_and_raise()
sys.stdout.write('\n')
queue.maxsize += len(threads)
for t in threads:
queue.put(None)
for t in threads:
t.join_and_raise()
