def download(client, container, object, path): res = client.retrieve_object_hashmap(container, object) blocksize = int(res['block_size']) blockhash = res['block_hash'] bytes = res['bytes'] map = res['hashes'] if os.path.exists(path): h = HashMap(blocksize, blockhash) h.load(open(path)) hashes = [hexlify(x) for x in h] else: open(path, 'w').close() # Create an empty file hashes = [] with open(path, 'a+') as fp: if bytes != 0: for i, h in enumerate(map): if i < len(hashes) and h == hashes[i]: continue start = i * blocksize end = '' if i == len(map) - 1 else ((i + 1) * blocksize) - 1 data = client.retrieve_object( container, object, range='bytes=%s-%s' % (start, end)) if i != len(map) - 1: data += (blocksize - len(data)) * '\x00' fp.seek(start) fp.write(data) fp.truncate(bytes)
def upload(client, path, container, prefix, name=None, mimetype=None): meta = client.retrieve_container_metadata(container) blocksize = int(meta['x-container-block-size']) blockhash = meta['x-container-block-hash'] size = os.path.getsize(path) hashes = HashMap(blocksize, blockhash) hashes.load(open(path)) map = {'bytes': size, 'hashes': [hexlify(x) for x in hashes]} objectname = name if name else os.path.split(path)[-1] object = prefix + objectname kwargs = {'mimetype': mimetype} if mimetype else {} v = None try: v = client.create_object_by_hashmap(container, object, map, **kwargs) except Fault, fault: if fault.status != 409: raise
class test_hashmap(unittest.TestCase): """ Test class for class Hashmap In all the quadratic probing tests unexpected errors can occur, for example if size of hashmap is small even one element can fill up a significant amount of size of the hashmap. Therefore I didn't use loop testing in quadratic probing it was causing random errors! """ def setUp(self): size = math.floor( random.random() * 30) + 30 # minimum size 15 to avoid coflicts with line 26 self._dict = HashMap(size, probing='quadratic', verbose=False) def test_hash_index(self): idx = self._dict.hash_index('divyansh') verify = 'divyansh'.__hash__() & ( self._dict.size - 1 ) # __hash__ returns different hashes for different python runs self.assertEqual(idx, verify) def test_sanity_check_key(self): self.assertRaises(ValueError, self._dict.sanity_check_key, 23) # passed any number to raise exception self.assertRaises(ValueError, self._dict.sanity_check_key, [2, 3, 4]) # passed a list to raise exception self.assertRaises(ValueError, self._dict.sanity_check_key, ('asd', 'asd')) # passed a tuple to raise exception def test_probe_linear(self): self._dict.probing = 'linear' num = self._dict.probe(5) self.assertEqual(num, 6) self._dict.ctr = 1 num = self._dict.probe(self._dict.size + 3) self.assertEqual(num, 4) def test_probe_quadratic(self): self._dict.probing = 'quadratic' self.assertEqual(self._dict.probe(5), 6) self.assertEqual(self._dict.probe(6), 10) self.assertEqual(self._dict.probe(1), 10) self._dict.reset() self.assertEqual(self._dict.probe(self._dict.size - 1), 0) self.assertEqual(self._dict.probe(self._dict.size - 1), 3) def test_set_quadratic(self): self.assertEqual(self._dict.set('animal', 'monkey'), True) self.assertEqual(self._dict.set('animal', 'lion'), True) self.assertRaises(ValueError, self._dict.set, 23, 'twenty_three') def test_set_linear(self): self._dict.probing = 'linear' self.assertEqual(self._dict.set('animal', 'monkey'), True) self.assertEqual(self._dict.set('animal', 'lion'), True) self.assertRaises(ValueError, self._dict.set, 23, 'twenty_three') self._dict.reset() for i in range(self._dict.size): self.assertEqual( self._dict.set('animal' + str(i), 'monkey' + str(i)), True) self.assertEqual(self._dict.load(), 1) self.assertEqual( self._dict.set('one_more_key_to_cause_overflow', 'monkey_n'), False) def test_get_quadratic(self): self.probing = 'quadratic' self.assertEqual(self._dict.set('animal' + str(3), 'monkey' + str(3)), True) self.assertEqual(self._dict.get('animal' + str(3)), 'monkey' + str(3)) self.assertRaises(KeyError, self._dict.get, 'bird') def test_get_linear(self): self._dict.probing = 'linear' for i in range(self._dict.size): self.assertEqual( self._dict.set('animal' + str(i), 'monkey' + str(i)), True) for i in reversed(range(self._dict.size)): self.assertEqual(self._dict.get('animal' + str(i)), 'monkey' + str(i)) self.assertRaises(KeyError, self._dict.get, 'bird') def test_delete_quadratic(self): self._dict.probing = 'quadratic' self._dict.set('animal', 'monkey') self._dict.delete('animal') self.assertRaises(KeyError, self._dict.get, 'animal') def test_delete_linear(self): self._dict.probing = 'linear' self._dict.set('animal', 'monkey') self._dict.delete('animal') for i in range(self._dict.size): self.assertEqual( self._dict.set('animal' + str(i), 'monkey' + str(i)), True) for i in reversed(range(self._dict.size)): self.assertEqual(self._dict.delete('animal' + str(i)), None) self.assertRaises(KeyError, self._dict.delete, 'animal' + str(i)) self.assertRaises(KeyError, self._dict.get, 'animal') def test_load_quadratic(self): self.probing = 'quadratic' itr = math.floor(random.random() * self._dict.size / 2 + 5) for i in range(itr): random_key = ''.join( random.choices(string.ascii_uppercase + string.digits, k=15)) self._dict.set(random_key, 'monkey' + str(i)) load_value = self._dict.load() self.assertLessEqual(load_value, 0.6) def test_load_linear(self): self.probing = 'linear' itr = math.floor(random.random() * self._dict.size + 5) for i in range(itr): random_key = ''.join( random.choices(string.ascii_uppercase + string.digits, k=15)) self._dict.set(random_key, 'monkey' + str(i)) load_value = self._dict.load() self.assertLessEqual(load_value, 1)