def __init__(self, context):
super().__init__()
self.context = context
self.logger = logging.getLogger(utils.logger_str(__class__) \
+ " " + context)
# self.logger.setLevel(logging.INFO)
self.config = config.Config.instance()
self.copies = int(self.config.get(self.context, "copies", 2))
self.path = config.path_for(self.config.get(self.context, "source"))
self.scanner = scanner.Scanner(self.context, self.path)
lazy_write = utils.str_to_duration(
self.config.get(context, "LAZY WRITE", 5))
# TODO: support expiration
self.rescan = utils.str_to_duration(
self.config.get(self.context, "rescan"))
self.clients = persistent_dict.PersistentDict(
f"/tmp/cb.s{context}.json.bz2",
lazy_write=lazy_write,
cls=lock.Lock,
expiry=self.rescan)
self.drains = elapsed.ExpiringDict(300) # NOT persistent!
self.locks = locker.Locker(5)
# TODO: timers should relate to a configurable cycle time
self.bailout = False
self.stats = {'claims': 0, 'drops': 0}
self.handling = False
def test_durations(self):
self.assertEquals(utils.str_to_duration("42"), 42)
self.assertEquals(utils.str_to_duration("60m"), 60*60)
self.assertEquals(utils.str_to_duration("27h5m"), 27*3600+5*60)
self.assertEquals(utils.str_to_duration("2d5m8"), 2*24*3600+5*60+8)
self.assertEquals(utils.duration_to_str(60*60),"60m")
self.assertEquals(utils.duration_to_str(3*60*60+11),"3h11s")
self.assertEquals(utils.duration_to_str(8*60+3*60*60*24+11),"3d8m11s")
def get_interval(self, interval_name):
interval = utils.str_to_duration( \
self.config.get(self.context, interval_name))
for source_context in self.random_source_list:
interval = min(interval,
utils.str_to_duration( \
self.config.get(source_context, interval_name)))
return interval
def __init__(self, context):
super().__init__()
self.context = context
self.config = config.Config.instance()
self.logger = logging.getLogger(logger_str(__class__) + " " + context)
self.logger.info(f"Creating clientlet {self.context}")
self.path = config.path_for(self.config.get(self.context, "backup"))
assert os.path.exists(self.path), f"{self.path} does not exist!"
# ALL source contexts (we care a lot)
self.sources = {}
self.scanners = {}
self.random_source_list = []
self.build_sources()
lazy_write = utils.str_to_duration(
self.config.get(context, "LAZY WRITE", 5))
# TODO: my cache of claims should expire in rescan/2
self.rescan = self.get_interval("rescan") // 2
self.claims = PersistentDict(f"/tmp/cb.c{context}.json.bz2",
lazy_write=5,
expiry=self.rescan)
self.drops = 0 # count the number of times I drop a file
self.stats = stats.Stats()
self.update_allocation()
self.bailing = False
self.datagrams = {}
def __init__(self, context, path, **kwargs):
self.context = context
self.path = os.path.expanduser(path)
if "name" in kwargs:
name = kwargs["name"]
else:
name = context
if "checksums" in kwargs:
self.checksums = kwargs['checksums']
else:
self.checksums = True
self.config = config.Config.instance()
self.pd_filename = f".cb.{context}.json.bz2"
lazy_write = utils.str_to_duration(
self.config.get(context, "LAZY WRITE", 5))
super().__init__(f"{self.path}/{self.pd_filename}",
lazy_write=lazy_write,
**kwargs)
self.logger = logging.getLogger(logger_str(__class__) + " " + name)
# self.logger.setLevel(logging.INFO)
self.ignored_suffixes = {}
self.report_timer = elapsed.ElapsedTimer()
self.stat = stats.Statistic(buckets=(0, 5, 10, 30))
def state_latency(self, states):
latency = time.time() - self.newest_checksum(states)
if latency < str_to_duration(self.config.getOption("cycle", "24h")):
msg = "Current"
else:
msg = "Stale"
msg += f"; last update {duration_to_str(latency)} ago"
return msg
def start(self):
while True:
self.logger.info("Starting")
self.config.load()
sources = self.config.get_sources_for_host(self.hostname)
print(f"sources: {sources}")
if len(sources.items()) > 0:
for context, source in sources.items():
self.logger.info(f"{context}: {source}")
gcm = GhettoClusterSource(context)
gcm.scan()
self.get_status(context, source, False)
self.logger.info("sources are complete.")
else:
self.logger.info("source of None")
replicas = self.config.get_replicas_for_host(self.hostname)
if len(replicas.items()) > 0:
for context, replica in replicas.items():
self.logger.info(f"{context}: {replica}")
source = self.config.get_source_for_context(context)
dest = config.path_for(replica)
gcs = GhettoClusterReplica(context, replica, source)
# gcs.pull()
# gcs.scan()
puller = Thread(target=gcs.pull)
self.logger.info("Starting pull thread")
puller.start()
timer = elapsed.ElapsedTimer()
while puller.is_alive():
if timer.once_every(15):
scanner = Thread(target=gcs.scan)
self.logger.debug("Starting scan thread")
scanner.start()
scanner.join()
self.logger.debug("Scan thread finished")
else:
time.sleep(1) # spin, but not hard
gcs.scan()
self.logger.info("Replicas are complete")
else:
self.logger.info("replica to noone")
try:
signal.signal(signal.SIGHUP, self.wakeup)
CYCLE = str_to_duration(self.config.getOption("CYCLE", "24h"))
self.logger.info(f"Sleeping for {duration_to_str(CYCLE)}" + \
f" in PID {os.getpid()}")
self.logger.debug("send SIGHUP to wake up")
time.sleep(CYCLE)
except WakeupException:
self.logger.warn(f"Restarting as requested (SIGHUP)")
signal.signal(signal.SIGHUP, signal.SIG_DFL)
def run(self):
self.bailout = False
# TODO: turbo prescan
self.scanner.scan(turbo=True)
self.logger.info("Ready to serve")
self.handling = True
while not self.bailout:
self.config.load()
self.scanner.scan()
self.update_files()
self.heartbeat()
sleep_time = utils.str_to_duration( \
self.config.get(self.context, "rescan"))
self.logger.info("Scanning complete; will re-scan in " \
+ f"{utils.duration_to_str(sleep_time)}")
time.sleep(sleep_time)
def run_forever(self, deleting=False):
try:
signal.signal(signal.SIGHUP, self.wakeup)
signal.signal(signal.SIGTERM, self.go_peacefully)
while True:
self.run(deleting=deleting)
CYCLE = str_to_duration(self.config.getOption("cycle", "24h"))
self.logger.info(f"Sleeping for {duration_to_str(CYCLE)}" + \
f" in PID {os.getpid()}")
self.logger.debug("send SIGHUP to wake up")
time.sleep(CYCLE)
except WakeupException:
self.logger.warn(f"Restarting as requested (SIGHUP)")
signal.signal(signal.SIGHUP, signal.SIG_DFL)
except TermException:
self.logger.warn(f"Exiting for SIGTERM")
sys.exit()
except KeyboardInterrupt:
self.logger.warn(f" Exiting...")
sys.exit()
def __init__(self, context):
super().__init__()
self.context = context
logger_str = f"{utils.logger_str(__class__)} {context}"
self.logger = logging.getLogger(logger_str)
# self.logger.setLevel(logging.INFO)
self.config = config.Config.instance()
self.copies = int(self.config.get(self.context, "copies", 2))
self.path = config.path_for(self.config.get(self.context, "source"))
self.scanner = scanner.ScannerLite(self.context, self.path)
self.rescan = utils.get_interval(self.config, "rescan", self.context)
lazy_write = self.config.get(context, "LAZY WRITE", 5)
lazy_write = utils.str_to_duration(lazy_write)
# self.clients: { filename : { client: expiry_time, } }
clients_state = f"/tmp/cb.{context}-clients.json.bz2"
self.clients = PersistentDict(clients_state, lazy_write=5)
self.stats = stats.Stats()
self.handling = False
def replica_is_current(self, latency):
return latency < str_to_duration(self.config.getOption("cycle", "24h"))