def __init__( self, server, statefile, historyfile, expiration_enabled, mode, override_lease_duration, # used if expiration_mode=="age" cutoff_date, # used if expiration_mode=="cutoff-date" sharetypes): self.historyfile = historyfile self.expiration_enabled = expiration_enabled self.mode = mode self.override_lease_duration = None self.cutoff_date = None if self.mode == "age": assert isinstance(override_lease_duration, (int, type(None))) self.override_lease_duration = override_lease_duration # seconds elif self.mode == "cutoff-date": assert isinstance(cutoff_date, int) # seconds-since-epoch assert cutoff_date is not None self.cutoff_date = cutoff_date else: raise ValueError("GC mode '%s' must be 'age' or 'cutoff-date'" % mode) self.sharetypes_to_expire = sharetypes ShareCrawler.__init__(self, server, statefile)
def test_empty_subclass(self): self.basedir = "crawler/Basic/empty_subclass" fileutil.make_dirs(self.basedir) serverid = "\x00" * 20 ss = StorageServer(self.basedir, serverid) ss.setServiceParent(self.s) for i in range(10): self.write(i, ss, serverid) statefile = os.path.join(self.basedir, "statefile") c = ShareCrawler(ss, statefile) c.slow_start = 0 c.setServiceParent(self.s) # we just let it run for a while, to get figleaf coverage of the # empty methods in the base class def _check(): return bool(c.state["last-cycle-finished"] is not None) d = self.poll(_check) def _done(ignored): state = c.get_state() self.failUnless(state["last-cycle-finished"] is not None) d.addCallback(_done) return d
def __init__(self, server, statefile, historyfile, expiration_enabled, mode, override_lease_duration, # used if expiration_mode=="age" cutoff_date, # used if expiration_mode=="cutoff-date" sharetypes): self.historyfile = historyfile self.expiration_enabled = expiration_enabled self.mode = mode self.override_lease_duration = None self.cutoff_date = None if self.mode == "age": assert isinstance(override_lease_duration, (int, type(None))) self.override_lease_duration = override_lease_duration # seconds elif self.mode == "cutoff-date": assert isinstance(cutoff_date, int) # seconds-since-epoch assert cutoff_date is not None self.cutoff_date = cutoff_date else: raise ValueError("GC mode '%s' must be 'age' or 'cutoff-date'" % mode) self.sharetypes_to_expire = sharetypes ShareCrawler.__init__(self, server, statefile)
def get_state(self): """In addition to the crawler state described in ShareCrawler.get_state(), I return the following keys which are specific to the lease-checker/expirer. Note that the non-history keys (with 'cycle' in their names) are only present if a cycle is currently running. If the crawler is between cycles, it appropriate to show the latest item in the 'history' key instead. Also note that each history item has all the data in the 'cycle-to-date' value, plus cycle-start-finish-times. cycle-to-date: expiration-enabled configured-expiration-mode lease-age-histogram (list of (minage,maxage,sharecount) tuples) leases-per-share-histogram corrupt-shares (list of (si_b32,shnum) tuples, minimal verification) space-recovered estimated-remaining-cycle: # Values may be None if not enough data has been gathered to # produce an estimate. space-recovered estimated-current-cycle: # cycle-to-date plus estimated-remaining. Values may be None if # not enough data has been gathered to produce an estimate. space-recovered history: maps cyclenum to a dict with the following keys: cycle-start-finish-times expiration-enabled configured-expiration-mode lease-age-histogram leases-per-share-histogram corrupt-shares space-recovered The 'space-recovered' structure is a dictionary with the following keys: # 'examined' is what was looked at examined-buckets, examined-buckets-mutable, examined-buckets-immutable examined-shares, -mutable, -immutable examined-sharebytes, -mutable, -immutable examined-diskbytes, -mutable, -immutable # 'actual' is what was actually deleted actual-buckets, -mutable, -immutable actual-shares, -mutable, -immutable actual-sharebytes, -mutable, -immutable actual-diskbytes, -mutable, -immutable # would have been deleted, if the original lease timer was used original-buckets, -mutable, -immutable original-shares, -mutable, -immutable original-sharebytes, -mutable, -immutable original-diskbytes, -mutable, -immutable # would have been deleted, if our configured max_age was used configured-buckets, -mutable, -immutable configured-shares, -mutable, -immutable configured-sharebytes, -mutable, -immutable configured-diskbytes, -mutable, -immutable """ progress = self.get_progress() state = ShareCrawler.get_state(self) # does a shallow copy history = pickle.load(open(self.historyfile, "rb")) state["history"] = history if not progress["cycle-in-progress"]: del state["cycle-to-date"] return state so_far = state["cycle-to-date"].copy() state["cycle-to-date"] = so_far lah = so_far["lease-age-histogram"] so_far["lease-age-histogram"] = self.convert_lease_age_histogram(lah) so_far["expiration-enabled"] = self.expiration_enabled so_far["configured-expiration-mode"] = (self.mode, self.override_lease_duration, self.cutoff_date, self.sharetypes_to_expire) so_far_sr = so_far["space-recovered"] remaining_sr = {} remaining = {"space-recovered": remaining_sr} cycle_sr = {} cycle = {"space-recovered": cycle_sr} if progress["cycle-complete-percentage"] > 0.0: pc = progress["cycle-complete-percentage"] / 100.0 m = (1 - pc) / pc for a in ("actual", "original", "configured", "examined"): for b in ("buckets", "shares", "sharebytes", "diskbytes"): for c in ("", "-mutable", "-immutable"): k = a + "-" + b + c remaining_sr[k] = m * so_far_sr[k] cycle_sr[k] = so_far_sr[k] + remaining_sr[k] else: for a in ("actual", "original", "configured", "examined"): for b in ("buckets", "shares", "sharebytes", "diskbytes"): for c in ("", "-mutable", "-immutable"): k = a + "-" + b + c remaining_sr[k] = None cycle_sr[k] = None state["estimated-remaining-cycle"] = remaining state["estimated-current-cycle"] = cycle return state
def __init__(self, *args, **kwargs): ShareCrawler.__init__(self, *args, **kwargs) self.counter = 0 self.finished_d = defer.Deferred()
def __init__(self, *args, **kwargs): ShareCrawler.__init__(self, *args, **kwargs) self.accumulated = 0.0 self.cycles = 0 self.last_yield = 0.0
def __init__(self, *args, **kwargs): ShareCrawler.__init__(self, *args, **kwargs) self.countdown = 6 self.all_buckets = [] self.finished_d = defer.Deferred() self.yield_cb = None
def __init__(self, *args, **kwargs): ShareCrawler.__init__(self, *args, **kwargs) self.all_buckets = [] self.finished_d = defer.Deferred()
def get_state(self): """In addition to the crawler state described in ShareCrawler.get_state(), I return the following keys which are specific to the lease-checker/expirer. Note that the non-history keys (with 'cycle' in their names) are only present if a cycle is currently running. If the crawler is between cycles, it appropriate to show the latest item in the 'history' key instead. Also note that each history item has all the data in the 'cycle-to-date' value, plus cycle-start-finish-times. cycle-to-date: expiration-enabled configured-expiration-mode lease-age-histogram (list of (minage,maxage,sharecount) tuples) leases-per-share-histogram corrupt-shares (list of (si_b32,shnum) tuples, minimal verification) space-recovered estimated-remaining-cycle: # Values may be None if not enough data has been gathered to # produce an estimate. space-recovered estimated-current-cycle: # cycle-to-date plus estimated-remaining. Values may be None if # not enough data has been gathered to produce an estimate. space-recovered history: maps cyclenum to a dict with the following keys: cycle-start-finish-times expiration-enabled configured-expiration-mode lease-age-histogram leases-per-share-histogram corrupt-shares space-recovered The 'space-recovered' structure is a dictionary with the following keys: # 'examined' is what was looked at examined-buckets, examined-buckets-mutable, examined-buckets-immutable examined-shares, -mutable, -immutable examined-sharebytes, -mutable, -immutable examined-diskbytes, -mutable, -immutable # 'actual' is what was actually deleted actual-buckets, -mutable, -immutable actual-shares, -mutable, -immutable actual-sharebytes, -mutable, -immutable actual-diskbytes, -mutable, -immutable # would have been deleted, if the original lease timer was used original-buckets, -mutable, -immutable original-shares, -mutable, -immutable original-sharebytes, -mutable, -immutable original-diskbytes, -mutable, -immutable # would have been deleted, if our configured max_age was used configured-buckets, -mutable, -immutable configured-shares, -mutable, -immutable configured-sharebytes, -mutable, -immutable configured-diskbytes, -mutable, -immutable """ progress = self.get_progress() state = ShareCrawler.get_state(self) # does a shallow copy history = pickle.load(open(self.historyfile, "rb")) state["history"] = history if not progress["cycle-in-progress"]: del state["cycle-to-date"] return state so_far = state["cycle-to-date"].copy() state["cycle-to-date"] = so_far lah = so_far["lease-age-histogram"] so_far["lease-age-histogram"] = self.convert_lease_age_histogram(lah) so_far["expiration-enabled"] = self.expiration_enabled so_far["configured-expiration-mode"] = (self.mode, self.override_lease_duration, self.cutoff_date, self.sharetypes_to_expire) so_far_sr = so_far["space-recovered"] remaining_sr = {} remaining = {"space-recovered": remaining_sr} cycle_sr = {} cycle = {"space-recovered": cycle_sr} if progress["cycle-complete-percentage"] > 0.0: pc = progress["cycle-complete-percentage"] / 100.0 m = (1-pc)/pc for a in ("actual", "original", "configured", "examined"): for b in ("buckets", "shares", "sharebytes", "diskbytes"): for c in ("", "-mutable", "-immutable"): k = a+"-"+b+c remaining_sr[k] = m * so_far_sr[k] cycle_sr[k] = so_far_sr[k] + remaining_sr[k] else: for a in ("actual", "original", "configured", "examined"): for b in ("buckets", "shares", "sharebytes", "diskbytes"): for c in ("", "-mutable", "-immutable"): k = a+"-"+b+c remaining_sr[k] = None cycle_sr[k] = None state["estimated-remaining-cycle"] = remaining state["estimated-current-cycle"] = cycle return state