def test_good_share_hosts(self): self.basedir = "checker/BalancingAct/1115" self.set_up_grid(num_servers=1) c0 = self.g.clients[0] c0.encoding_params['happy'] = 1 c0.encoding_params['n'] = 4 c0.encoding_params['k'] = 3 DATA = "data" * 100 d = c0.upload(Data(DATA, convergence="")) def _stash_immutable(ur): self.imm = c0.create_node_from_uri(ur.get_uri()) self.uri = self.imm.get_uri() d.addCallback(_stash_immutable) d.addCallback(lambda ign: self.find_uri_shares(self.uri)) def _store_shares(shares): self.shares = shares d.addCallback(_store_shares) def add_three(_, i): # Add a new server with just share 3 self.add_server_with_share(i, self.uri, 3) #print self._pretty_shares_chart(self.uri) for i in range(1, 5): d.addCallback(add_three, i) def _check_and_repair(_): return self.imm.check_and_repair(Monitor()) def _check_counts(crr, shares_good, good_share_hosts): prr = crr.get_post_repair_results() #print self._pretty_shares_chart(self.uri) self.failUnlessEqual(prr.get_share_counter_good(), shares_good) self.failUnlessEqual(prr.get_host_counter_good_shares(), good_share_hosts) """ Initial sharemap: 0:[A] 1:[A] 2:[A] 3:[A,B,C,D,E] 4 good shares, but 5 good hosts After deleting all instances of share #3 and repairing: 0:[A,B], 1:[A,C], 2:[A,D], 3:[E] Still 4 good shares and 5 good hosts """ d.addCallback(_check_and_repair) d.addCallback(_check_counts, 4, 5) d.addCallback(lambda _: self.delete_shares_numbered(self.uri, [3])) d.addCallback(_check_and_repair) d.addCallback(_check_counts, 4, 5) d.addCallback( lambda _: [self.g.break_server(sid) for sid in self.g.get_all_serverids()]) d.addCallback(_check_and_repair) d.addCallback(_check_counts, 0, 0) return d
def create_immutable_directory(self, children, convergence=None): if convergence is None: convergence = self.secret_holder.get_convergence_secret() packed = pack_children(children, None, deep_immutable=True) uploadable = Data(packed, convergence) d = self.uploader.upload(uploadable, history=self.history) d.addCallback(lambda results: self.create_from_cap(None, results.uri)) d.addCallback(self._create_dirnode) return d
def _start(ign): self.set_up_grid(num_servers=4) self.c0 = self.g.clients[0] self.c0.encoding_params = { "k": 1, "happy": 4, "n": 4, "max_segment_size": 5, } self.uris = {} DATA = "data" * 100 # 400/5 = 80 blocks return self.c0.upload(Data(DATA, convergence=""))
def test_875(self): self.basedir = "checker/AddLease/875" self.set_up_grid(num_servers=1) c0 = self.g.clients[0] c0.encoding_params['happy'] = 1 self.uris = {} DATA = "data" * 100 d = c0.upload(Data(DATA, convergence="")) def _stash_immutable(ur): self.imm = c0.create_node_from_uri(ur.get_uri()) d.addCallback(_stash_immutable) d.addCallback( lambda ign: c0.create_mutable_file(MutableData("contents"))) def _stash_mutable(node): self.mut = node d.addCallback(_stash_mutable) def _check_cr(cr, which): self.failUnless(cr.is_healthy(), which) # these two should work normally d.addCallback(lambda ign: self.imm.check(Monitor(), add_lease=True)) d.addCallback(_check_cr, "immutable-normal") d.addCallback(lambda ign: self.mut.check(Monitor(), add_lease=True)) d.addCallback(_check_cr, "mutable-normal") really_did_break = [] # now break the server's remote_add_lease call def _break_add_lease(ign): def broken_add_lease(*args, **kwargs): really_did_break.append(1) raise KeyError("intentional failure, should be ignored") assert self.g.servers_by_number[0].remote_add_lease self.g.servers_by_number[0].remote_add_lease = broken_add_lease d.addCallback(_break_add_lease) # and confirm that the files still look healthy d.addCallback(lambda ign: self.mut.check(Monitor(), add_lease=True)) d.addCallback(_check_cr, "mutable-broken") d.addCallback(lambda ign: self.imm.check(Monitor(), add_lease=True)) d.addCallback(_check_cr, "immutable-broken") d.addCallback(lambda ign: self.failUnless(really_did_break)) return d
def startup(self, basedir): self.basedir = basedir self.set_up_grid(num_clients=2, num_servers=5) c1 = self.g.clients[1] # We need multiple segments to test crypttext hash trees that are # non-trivial (i.e. they have more than just one hash in them). c1.encoding_params['max_segment_size'] = 12 # Tests that need to test servers of happiness using this should # set their own value for happy -- the default (7) breaks stuff. c1.encoding_params['happy'] = 1 d = c1.upload(Data(TEST_DATA, convergence="")) def _after_upload(ur): self.uri = ur.get_uri() self.filenode = self.g.clients[0].create_node_from_uri(ur.get_uri()) return self.uri d.addCallback(_after_upload) return d
def test_upload(self): basedir = "no_network/Harness/upload" g = self.grid(basedir) g.setServiceParent(self.s) c0 = g.clients[0] DATA = "Data to upload" * 100 data = Data(DATA, "") d = c0.upload(data) def _uploaded(res): n = c0.create_node_from_uri(res.get_uri()) return download_to_data(n) d.addCallback(_uploaded) def _check(res): self.failUnlessEqual(res, DATA) d.addCallback(_check) return d
def test_good_share_hosts(self): self.basedir = "checker/BalancingAct/1115" self.set_up_grid(num_servers=1) c0 = self.g.clients[0] c0.encoding_params['happy'] = 1 c0.encoding_params['n'] = 4 c0.encoding_params['k'] = 3 DATA = "data" * 100 d = c0.upload(Data(DATA, convergence="")) def _stash_immutable(ur): self.imm = c0.create_node_from_uri(ur.get_uri()) self.uri = self.imm.get_uri() d.addCallback(_stash_immutable) d.addCallback(lambda ign: self.find_uri_shares(self.uri)) def _store_shares(shares): self.shares = shares d.addCallback(_store_shares) def add_three(_, i): # Add a new server with just share 3 self.add_server_with_share(i, self.uri, 3) #print self._pretty_shares_chart(self.uri) for i in range(1, 5): d.addCallback(add_three, i) def _check_and_repair(_): return self.imm.check_and_repair(Monitor()) def _check_counts(crr, shares_good, good_share_hosts): prr = crr.get_post_repair_results() self.failUnlessEqual(prr.get_share_counter_good(), shares_good) self.failUnlessEqual(prr.get_host_counter_good_shares(), good_share_hosts) """ Initial sharemap: 0:[A] 1:[A] 2:[A] 3:[A,B,C,D,E] 4 good shares, but 5 good hosts After deleting all instances of share #3 and repairing: 0:[A], 1:[A,B], 2:[C,A], 3:[E] # actually: {0: ['E', 'A'], 1: ['C', 'A'], 2: ['A', 'B'], 3: ['D']} Still 4 good shares but now 4 good hosts """ d.addCallback(_check_and_repair) d.addCallback(_check_counts, 4, 5) d.addCallback(lambda _: self.delete_shares_numbered(self.uri, [3])) d.addCallback(_check_and_repair) # it can happen that our uploader will choose, e.g., to upload # to servers B, C, D, E .. which will mean that all 5 serves # now contain our shares (and thus "respond"). def _check_happy(crr): prr = crr.get_post_repair_results() self.assertTrue(prr.get_host_counter_good_shares() >= 4) return crr d.addCallback(_check_happy) d.addCallback(lambda _: all( [self.g.break_server(sid) for sid in self.g.get_all_serverids()])) d.addCallback(_check_and_repair) d.addCallback(_check_counts, 0, 0) return d
def _maybe_upload(ign, now=None): self._log("_maybe_upload: relpath_u=%r, now=%r" % (relpath_u, now)) if now is None: now = time.time() fp = self._get_filepath(relpath_u) pathinfo = get_pathinfo(unicode_from_filepath(fp)) self._log("about to remove %r from pending set %r" % (relpath_u, self._pending)) try: self._pending.remove(relpath_u) except KeyError: self._log("WRONG that %r wasn't in pending" % (relpath_u,)) encoded_path_u = magicpath.path2magic(relpath_u) if not pathinfo.exists: # FIXME merge this with the 'isfile' case. self._log("notified object %s disappeared (this is normal)" % quote_filepath(fp)) self._count('objects_disappeared') db_entry = self._db.get_db_entry(relpath_u) if db_entry is None: return False last_downloaded_timestamp = now # is this correct? if is_new_file(pathinfo, db_entry): new_version = db_entry.version + 1 else: self._log("Not uploading %r" % (relpath_u,)) self._count('objects_not_uploaded') return False metadata = { 'version': new_version, 'deleted': True, 'last_downloaded_timestamp': last_downloaded_timestamp, } if db_entry.last_downloaded_uri is not None: metadata['last_downloaded_uri'] = db_entry.last_downloaded_uri empty_uploadable = Data("", self._client.convergence) d2 = self._upload_dirnode.add_file( encoded_path_u, empty_uploadable, metadata=metadata, overwrite=True, progress=item.progress, ) def _add_db_entry(filenode): filecap = filenode.get_uri() last_downloaded_uri = metadata.get('last_downloaded_uri', None) self._db.did_upload_version(relpath_u, new_version, filecap, last_downloaded_uri, last_downloaded_timestamp, pathinfo) self._count('files_uploaded') d2.addCallback(_add_db_entry) d2.addCallback(lambda ign: True) return d2 elif pathinfo.islink: self.warn("WARNING: cannot upload symlink %s" % quote_filepath(fp)) return False elif pathinfo.isdir: self._log("ISDIR") if not getattr(self._notifier, 'recursive_includes_new_subdirectories', False): self._notifier.watch(fp, mask=self.mask, callbacks=[self._notify], recursive=True) db_entry = self._db.get_db_entry(relpath_u) self._log("isdir dbentry %r" % (db_entry,)) if not is_new_file(pathinfo, db_entry): self._log("NOT A NEW FILE") return False uploadable = Data("", self._client.convergence) encoded_path_u += magicpath.path2magic(u"/") self._log("encoded_path_u = %r" % (encoded_path_u,)) upload_d = self._upload_dirnode.add_file( encoded_path_u, uploadable, metadata={"version": 0}, overwrite=True, progress=item.progress, ) def _dir_succeeded(ign): self._log("created subdirectory %r" % (relpath_u,)) self._count('directories_created') def _dir_failed(f): self._log("failed to create subdirectory %r" % (relpath_u,)) return f upload_d.addCallbacks(_dir_succeeded, _dir_failed) upload_d.addCallback(lambda ign: self._scan(relpath_u)) upload_d.addCallback(lambda ign: True) return upload_d elif pathinfo.isfile: db_entry = self._db.get_db_entry(relpath_u) last_downloaded_timestamp = now if db_entry is None: new_version = 0 elif is_new_file(pathinfo, db_entry): new_version = db_entry.version + 1 else: self._log("Not uploading %r" % (relpath_u,)) self._count('objects_not_uploaded') return False metadata = { 'version': new_version, 'last_downloaded_timestamp': last_downloaded_timestamp, } if db_entry is not None and db_entry.last_downloaded_uri is not None: metadata['last_downloaded_uri'] = db_entry.last_downloaded_uri uploadable = FileName(unicode_from_filepath(fp), self._client.convergence) d2 = self._upload_dirnode.add_file( encoded_path_u, uploadable, metadata=metadata, overwrite=True, progress=item.progress, ) def _add_db_entry(filenode): filecap = filenode.get_uri() last_downloaded_uri = metadata.get('last_downloaded_uri', None) self._db.did_upload_version(relpath_u, new_version, filecap, last_downloaded_uri, last_downloaded_timestamp, pathinfo) self._count('files_uploaded') return True d2.addCallback(_add_db_entry) return d2 else: self.warn("WARNING: cannot process special file %s" % quote_filepath(fp)) return False