def __init__(
self,
server,
statefile,
historyfile,
expiration_enabled,
mode,
override_lease_duration, # used if expiration_mode=="age"
cutoff_date, # used if expiration_mode=="cutoff-date"
sharetypes):
self.historyfile = historyfile
self.expiration_enabled = expiration_enabled
self.mode = mode
self.override_lease_duration = None
self.cutoff_date = None
if self.mode == "age":
assert isinstance(override_lease_duration, (int, type(None)))
self.override_lease_duration = override_lease_duration # seconds
elif self.mode == "cutoff-date":
assert isinstance(cutoff_date, int) # seconds-since-epoch
assert cutoff_date is not None
self.cutoff_date = cutoff_date
else:
raise ValueError("GC mode '%s' must be 'age' or 'cutoff-date'" %
mode)
self.sharetypes_to_expire = sharetypes
ShareCrawler.__init__(self, server, statefile)
def test_empty_subclass(self):
self.basedir = "crawler/Basic/empty_subclass"
fileutil.make_dirs(self.basedir)
serverid = "\x00" * 20
ss = StorageServer(self.basedir, serverid)
ss.setServiceParent(self.s)
for i in range(10):
self.write(i, ss, serverid)
statefile = os.path.join(self.basedir, "statefile")
c = ShareCrawler(ss, statefile)
c.slow_start = 0
c.setServiceParent(self.s)
# we just let it run for a while, to get figleaf coverage of the
# empty methods in the base class
def _check():
return bool(c.state["last-cycle-finished"] is not None)
d = self.poll(_check)
def _done(ignored):
state = c.get_state()
self.failUnless(state["last-cycle-finished"] is not None)
d.addCallback(_done)
return d
def test_empty_subclass(self):
self.basedir = "crawler/Basic/empty_subclass"
fileutil.make_dirs(self.basedir)
serverid = "\x00" * 20
ss = StorageServer(self.basedir, serverid)
ss.setServiceParent(self.s)
for i in range(10):
self.write(i, ss, serverid)
statefile = os.path.join(self.basedir, "statefile")
c = ShareCrawler(ss, statefile)
c.slow_start = 0
c.setServiceParent(self.s)
# we just let it run for a while, to get figleaf coverage of the
# empty methods in the base class
def _check():
return bool(c.state["last-cycle-finished"] is not None)
d = self.poll(_check)
def _done(ignored):
state = c.get_state()
self.failUnless(state["last-cycle-finished"] is not None)
d.addCallback(_done)
return d
def __init__(self, server, statefile, historyfile,
expiration_enabled, mode,
override_lease_duration, # used if expiration_mode=="age"
cutoff_date, # used if expiration_mode=="cutoff-date"
sharetypes):
self.historyfile = historyfile
self.expiration_enabled = expiration_enabled
self.mode = mode
self.override_lease_duration = None
self.cutoff_date = None
if self.mode == "age":
assert isinstance(override_lease_duration, (int, type(None)))
self.override_lease_duration = override_lease_duration # seconds
elif self.mode == "cutoff-date":
assert isinstance(cutoff_date, int) # seconds-since-epoch
assert cutoff_date is not None
self.cutoff_date = cutoff_date
else:
raise ValueError("GC mode '%s' must be 'age' or 'cutoff-date'" % mode)
self.sharetypes_to_expire = sharetypes
ShareCrawler.__init__(self, server, statefile)
def get_state(self):
"""In addition to the crawler state described in
ShareCrawler.get_state(), I return the following keys which are
specific to the lease-checker/expirer. Note that the non-history keys
(with 'cycle' in their names) are only present if a cycle is
currently running. If the crawler is between cycles, it appropriate
to show the latest item in the 'history' key instead. Also note that
each history item has all the data in the 'cycle-to-date' value, plus
cycle-start-finish-times.
cycle-to-date:
expiration-enabled
configured-expiration-mode
lease-age-histogram (list of (minage,maxage,sharecount) tuples)
leases-per-share-histogram
corrupt-shares (list of (si_b32,shnum) tuples, minimal verification)
space-recovered
estimated-remaining-cycle:
# Values may be None if not enough data has been gathered to
# produce an estimate.
space-recovered
estimated-current-cycle:
# cycle-to-date plus estimated-remaining. Values may be None if
# not enough data has been gathered to produce an estimate.
space-recovered
history: maps cyclenum to a dict with the following keys:
cycle-start-finish-times
expiration-enabled
configured-expiration-mode
lease-age-histogram
leases-per-share-histogram
corrupt-shares
space-recovered
The 'space-recovered' structure is a dictionary with the following
keys:
# 'examined' is what was looked at
examined-buckets, examined-buckets-mutable, examined-buckets-immutable
examined-shares, -mutable, -immutable
examined-sharebytes, -mutable, -immutable
examined-diskbytes, -mutable, -immutable
# 'actual' is what was actually deleted
actual-buckets, -mutable, -immutable
actual-shares, -mutable, -immutable
actual-sharebytes, -mutable, -immutable
actual-diskbytes, -mutable, -immutable
# would have been deleted, if the original lease timer was used
original-buckets, -mutable, -immutable
original-shares, -mutable, -immutable
original-sharebytes, -mutable, -immutable
original-diskbytes, -mutable, -immutable
# would have been deleted, if our configured max_age was used
configured-buckets, -mutable, -immutable
configured-shares, -mutable, -immutable
configured-sharebytes, -mutable, -immutable
configured-diskbytes, -mutable, -immutable
"""
progress = self.get_progress()
state = ShareCrawler.get_state(self) # does a shallow copy
history = pickle.load(open(self.historyfile, "rb"))
state["history"] = history
if not progress["cycle-in-progress"]:
del state["cycle-to-date"]
return state
so_far = state["cycle-to-date"].copy()
state["cycle-to-date"] = so_far
lah = so_far["lease-age-histogram"]
so_far["lease-age-histogram"] = self.convert_lease_age_histogram(lah)
so_far["expiration-enabled"] = self.expiration_enabled
so_far["configured-expiration-mode"] = (self.mode,
self.override_lease_duration,
self.cutoff_date,
self.sharetypes_to_expire)
so_far_sr = so_far["space-recovered"]
remaining_sr = {}
remaining = {"space-recovered": remaining_sr}
cycle_sr = {}
cycle = {"space-recovered": cycle_sr}
if progress["cycle-complete-percentage"] > 0.0:
pc = progress["cycle-complete-percentage"] / 100.0
m = (1 - pc) / pc
for a in ("actual", "original", "configured", "examined"):
for b in ("buckets", "shares", "sharebytes", "diskbytes"):
for c in ("", "-mutable", "-immutable"):
k = a + "-" + b + c
remaining_sr[k] = m * so_far_sr[k]
cycle_sr[k] = so_far_sr[k] + remaining_sr[k]
else:
for a in ("actual", "original", "configured", "examined"):
for b in ("buckets", "shares", "sharebytes", "diskbytes"):
for c in ("", "-mutable", "-immutable"):
k = a + "-" + b + c
remaining_sr[k] = None
cycle_sr[k] = None
state["estimated-remaining-cycle"] = remaining
state["estimated-current-cycle"] = cycle
return state
def __init__(self, *args, **kwargs):
ShareCrawler.__init__(self, *args, **kwargs)
self.counter = 0
self.finished_d = defer.Deferred()
def __init__(self, *args, **kwargs):
ShareCrawler.__init__(self, *args, **kwargs)
self.accumulated = 0.0
self.cycles = 0
self.last_yield = 0.0
def __init__(self, *args, **kwargs):
ShareCrawler.__init__(self, *args, **kwargs)
self.countdown = 6
self.all_buckets = []
self.finished_d = defer.Deferred()
self.yield_cb = None
def __init__(self, *args, **kwargs):
ShareCrawler.__init__(self, *args, **kwargs)
self.all_buckets = []
self.finished_d = defer.Deferred()
def get_state(self):
"""In addition to the crawler state described in
ShareCrawler.get_state(), I return the following keys which are
specific to the lease-checker/expirer. Note that the non-history keys
(with 'cycle' in their names) are only present if a cycle is
currently running. If the crawler is between cycles, it appropriate
to show the latest item in the 'history' key instead. Also note that
each history item has all the data in the 'cycle-to-date' value, plus
cycle-start-finish-times.
cycle-to-date:
expiration-enabled
configured-expiration-mode
lease-age-histogram (list of (minage,maxage,sharecount) tuples)
leases-per-share-histogram
corrupt-shares (list of (si_b32,shnum) tuples, minimal verification)
space-recovered
estimated-remaining-cycle:
# Values may be None if not enough data has been gathered to
# produce an estimate.
space-recovered
estimated-current-cycle:
# cycle-to-date plus estimated-remaining. Values may be None if
# not enough data has been gathered to produce an estimate.
space-recovered
history: maps cyclenum to a dict with the following keys:
cycle-start-finish-times
expiration-enabled
configured-expiration-mode
lease-age-histogram
leases-per-share-histogram
corrupt-shares
space-recovered
The 'space-recovered' structure is a dictionary with the following
keys:
# 'examined' is what was looked at
examined-buckets, examined-buckets-mutable, examined-buckets-immutable
examined-shares, -mutable, -immutable
examined-sharebytes, -mutable, -immutable
examined-diskbytes, -mutable, -immutable
# 'actual' is what was actually deleted
actual-buckets, -mutable, -immutable
actual-shares, -mutable, -immutable
actual-sharebytes, -mutable, -immutable
actual-diskbytes, -mutable, -immutable
# would have been deleted, if the original lease timer was used
original-buckets, -mutable, -immutable
original-shares, -mutable, -immutable
original-sharebytes, -mutable, -immutable
original-diskbytes, -mutable, -immutable
# would have been deleted, if our configured max_age was used
configured-buckets, -mutable, -immutable
configured-shares, -mutable, -immutable
configured-sharebytes, -mutable, -immutable
configured-diskbytes, -mutable, -immutable
"""
progress = self.get_progress()
state = ShareCrawler.get_state(self) # does a shallow copy
history = pickle.load(open(self.historyfile, "rb"))
state["history"] = history
if not progress["cycle-in-progress"]:
del state["cycle-to-date"]
return state
so_far = state["cycle-to-date"].copy()
state["cycle-to-date"] = so_far
lah = so_far["lease-age-histogram"]
so_far["lease-age-histogram"] = self.convert_lease_age_histogram(lah)
so_far["expiration-enabled"] = self.expiration_enabled
so_far["configured-expiration-mode"] = (self.mode,
self.override_lease_duration,
self.cutoff_date,
self.sharetypes_to_expire)
so_far_sr = so_far["space-recovered"]
remaining_sr = {}
remaining = {"space-recovered": remaining_sr}
cycle_sr = {}
cycle = {"space-recovered": cycle_sr}
if progress["cycle-complete-percentage"] > 0.0:
pc = progress["cycle-complete-percentage"] / 100.0
m = (1-pc)/pc
for a in ("actual", "original", "configured", "examined"):
for b in ("buckets", "shares", "sharebytes", "diskbytes"):
for c in ("", "-mutable", "-immutable"):
k = a+"-"+b+c
remaining_sr[k] = m * so_far_sr[k]
cycle_sr[k] = so_far_sr[k] + remaining_sr[k]
else:
for a in ("actual", "original", "configured", "examined"):
for b in ("buckets", "shares", "sharebytes", "diskbytes"):
for c in ("", "-mutable", "-immutable"):
k = a+"-"+b+c
remaining_sr[k] = None
cycle_sr[k] = None
state["estimated-remaining-cycle"] = remaining
state["estimated-current-cycle"] = cycle
return state
def __init__(self, *args, **kwargs):
ShareCrawler.__init__(self, *args, **kwargs)
self.counter = 0
self.finished_d = defer.Deferred()
def __init__(self, *args, **kwargs):
ShareCrawler.__init__(self, *args, **kwargs)
self.accumulated = 0.0
self.cycles = 0
self.last_yield = 0.0
def __init__(self, *args, **kwargs):
ShareCrawler.__init__(self, *args, **kwargs)
self.countdown = 6
self.all_buckets = []
self.finished_d = defer.Deferred()
self.yield_cb = None
def __init__(self, *args, **kwargs):
ShareCrawler.__init__(self, *args, **kwargs)
self.all_buckets = []
self.finished_d = defer.Deferred()
