def test_minimum_assignments(self):
segment = sync.Segment('123456',
services=self.services,
rethinker=self.rethinker,
registry=self.registry,
size=100)
output = segment.minimum_assignments()
self.assertEqual(output, 1)
segment = sync.Segment('228188',
services=self.services,
rethinker=self.rethinker,
registry=self.registry,
size=100)
output = segment.minimum_assignments()
self.assertEqual(output, 2)
def test_copy_segment_from_hdfs(self, error, snakebite):
results = [{'error': 'test error'}]
class C:
def __init__(*args, **kwargs):
pass
def copyToLocal(self, paths, dst, *args, **kwargs):
for result in results:
if not result.get('error'):
# create empty dest file
with open(dst, 'wb') as f:
pass
yield result
snakebite.Client = C
controller = self.make_fresh_controller()
segment = sync.Segment('test-segment',
services=self.services,
rethinker=self.rethinker,
registry=self.registry,
size=100,
remote_path='/fake/remote/path')
with self.assertRaises(Exception):
output = controller.copy_segment_from_hdfs(segment)
results = [{}]
output = controller.copy_segment_from_hdfs(segment)
self.assertEqual(output, True)
def test_host_key(self):
segment = sync.Segment('test-segment',
services=self.services,
rethinker=self.rethinker,
registry=self.registry,
size=100)
key = segment.host_key('test-node')
self.assertEqual(key, 'test-node:test-segment')
def test_local_segment_exists(self):
segment = sync.Segment('123456',
services=self.services,
rethinker=self.rethinker,
registry=self.registry,
size=100)
output = segment.local_segment_exists()
self.assertEqual(output, False)
def test_local_path(self):
segment = sync.Segment('123456',
services=self.services,
rethinker=self.rethinker,
registry=self.registry,
size=100)
output = segment.local_path()
self.assertEqual(output,
os.path.join(settings['LOCAL_DATA'], '123456.sqlite'))
def test_provision_local_segment(self):
segment = sync.Segment('123456-test-database',
services=self.services,
rethinker=self.rethinker,
registry=self.registry,
size=100)
if segment.local_segment_exists():
os.remove(segment.local_path())
output = segment.provision_local_segment('')
os.remove(segment.local_path())
def test_assign(self):
registry = sync.HostRegistry(rethinker=self.rethinker,
services=self.services)
segment = sync.Segment('123456',
services=self.services,
rethinker=self.rethinker,
registry=registry,
size=1024)
registry.assign('localhost', segment, '/fake/path')
self.assertEqual(registry.assignment_queue._queue[0]['id'],
'localhost:123456')
return (segment, registry)
def test_new_write_lock(self):
lock = sync.Lock.load(self.rethinker, 'write:lock:123456')
if lock:
lock.release()
segment = sync.Segment('123456',
services=self.services,
rethinker=self.rethinker,
registry=self.registry,
size=100)
lock = segment.new_write_lock()
with self.assertRaises(Exception):
segment.new_write_lock()
lock.release()
def test_provision_writable_segment(self):
test_segment = sync.Segment('test',
services=self.services,
rethinker=self.rethinker,
registry=self.registry,
size=0)
test_path = test_segment.local_path()
if os.path.isfile(test_path):
os.remove(test_path)
called = []
controller = self.make_fresh_controller()
controller.provision_writable_segment('test')
self.assertEqual(os.path.isfile(test_path), True)
os.remove(test_path)
def test_retrieve_write_lock(self):
lock = sync.Lock.load(self.rethinker, 'write:lock:123456')
if lock:
lock.release()
segment = sync.Segment('123456',
services=self.services,
rethinker=self.rethinker,
registry=self.registry,
size=100)
output = segment.new_write_lock()
lock = segment.retrieve_write_lock()
self.assertEqual(lock["node"], settings['HOSTNAME'])
self.assertIn("acquired_on", lock)
lock.release()
def test_readable_copies(self):
registry = sync.HostRegistry(rethinker=self.rethinker,
services=self.services)
segment = sync.Segment('test-segment',
services=self.services,
rethinker=self.rethinker,
registry=self.registry,
size=100)
registry.heartbeat(pool='trough-read',
node=settings['HOSTNAME'],
ttl=0.4,
segment=segment.id)
output = segment.readable_copies()
output = list(output)
self.assertEqual(output[0]['node'], settings['HOSTNAME'])
def test_all_copies(self):
registry = sync.HostRegistry(rethinker=self.rethinker,
services=self.services)
segment = sync.Segment('test-segment',
services=self.services,
rethinker=self.rethinker,
registry=self.registry,
size=100)
registry.assign(hostname='test-pool',
segment=segment,
remote_path="/fake/path")
registry.commit_assignments()
output = segment.all_copies()
output = [item for item in output]
self.assertEqual(output[0]['id'], 'test-pool:test-segment')
def test_is_assigned_to_host(self):
segment = sync.Segment('test-segment',
services=self.services,
rethinker=self.rethinker,
registry=self.registry,
size=100)
registry = sync.HostRegistry(rethinker=self.rethinker,
services=self.services)
registry.assign(hostname='assigned',
segment=segment,
remote_path="/fake/path")
registry.commit_assignments()
output = segment.is_assigned_to_host('not-assigned')
self.assertFalse(output)
output = segment.is_assigned_to_host('assigned')
self.assertTrue(output)
def test_segments_for_host(self):
registry = sync.HostRegistry(rethinker=self.rethinker,
services=self.services)
segment = sync.Segment('123456',
services=self.services,
rethinker=self.rethinker,
registry=registry,
size=1024)
asmt = registry.assign('localhost', segment, '/fake/path')
registry.commit_assignments()
output = registry.segments_for_host('localhost')
self.assertEqual(output[0].id, '123456')
registry.unassign(asmt)
registry.commit_unassignments()
output = registry.segments_for_host('localhost')
self.assertEqual(output, [])