class DBInstanceRestful(): def __init__(self, config, persistence): self.config = config self.persistence = persistence self.controller = DBInstanceController(self.persistence) @cherrypy.expose @Helper.restful def add(self, id, host, port=3306, user="******", passwd="", data_dir="/var/lib/mysql"): if self.controller.get(id) is not None: raise Exception("duplicate instance") else: dbinstance = DBInstance(id, host, int(port), user, passwd, data_dir) self.controller.add(dbinstance) return "ok" @cherrypy.expose @Helper.restful def update(self, id, host, port=3306, user="******", passwd="", data_dir="/var/lib/mysql"): if self.controller.get(id) is None: raise Exception("instance not exist") else: dbinstance = DBInstance(id, host, int(port), user, passwd, data_dir) self.controller.update(dbinstance) return "ok" @cherrypy.expose @Helper.restful def delete(self, id): if self.controller.get(id) is None: raise Exception("instance not exist") else: self.controller.delete(id) return "ok" @cherrypy.expose @Helper.restful def exist(self, id): dbinstance = self.controller.get(id) if dbinstance is not None: return True else: return False
class DBReplicaRestful(): def __init__(self, config, persistence): self.config = config self.persistence = persistence self.controller = DBReplicaController(persistence) self.dbinstance_controller = DBInstanceController(persistence) self._init_worker() def _init_worker(self): self.workers = {} dbreplicas = self.controller.get_all() for dbreplica in dbreplicas: worker = ReplicaWorker(self.persistence, dbreplica, self.config) worker.start() self.workers[dbreplica.id] = worker @cherrypy.expose @Helper.restful def add(self, replica_id, name, master, slaves, check_period=60, binlog_window=0, no_slave_purge=1): if self.workers.has_key(replica_id): raise Exception("duplicate replication") else: dbreplica = DBReplica(replica_id, name, master, slaves, int(check_period), int(binlog_window), int(no_slave_purge)) worker = ReplicaWorker(self.persistence, dbreplica, self.config) worker.start() self.workers[replica_id] = worker self.controller.add(dbreplica) return "ok" @cherrypy.expose @Helper.restful def delete(self, replica_id): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: worker = self.workers.pop(replica_id) worker.stop() self.controller.delete(replica_id) return "ok" @cherrypy.expose @Helper.restful def add_slave(self, replica_id, slave_id): if not self.workers.has_key(replica_id): raise Exception("replication not exist") elif self.dbinstance_controller.get(slave_id) is None: raise Exception("no %s slave instance" % slave_id) else: self.controller.add_slave(replica_id, slave_id) self.workers[replica_id].stop() worker = ReplicaWorker(self.persistence, self.controller.get(replica_id), self.config) worker.start() self.workers[replica_id] = worker return "ok" @cherrypy.expose @Helper.restful def add_slaves(self, replica_id, slave_ids): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: slaves = json.loads(slave_ids) for slave_id in slaves: if self.dbinstance_controller.get(slave_id) is None: raise Exception("no %s slave instance" % slave_id) self.controller.add_slaves(replica_id,slaves) self.workers[replica_id].stop() worker = ReplicaWorker(self.persistence, self.controller.get(replica_id), self.config) worker.start() self.workers[replica_id] = worker return "ok" @cherrypy.expose @Helper.restful def del_slave(self, replica_id, slave_id): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: self.controller.del_slave(replica_id, slave_id) self.workers[replica_id].stop() worker = ReplicaWorker(self.persistence, self.controller.get(replica_id), self.config) worker.start() self.workers[replica_id] = worker return "ok" @cherrypy.expose @Helper.restful def update_check_period(self, replica_id, check_period): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: self.controller.update_check_period(replica_id, int(check_period)) self.workers[replica_id].stop() worker = ReplicaWorker(self.persistence, self.controller.get(replica_id), self.config) worker.start() self.workers[replica_id] = worker return "ok" @cherrypy.expose @Helper.restful def update_binlog_window(self, replica_id, binlog_window): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: self.controller.update_binlog_window(replica_id, int(binlog_window)) self.workers[replica_id].stop() worker = ReplicaWorker(self.persistence, self.controller.get(replica_id), self.config) worker.start() self.workers[replica_id] = worker return "ok" @cherrypy.expose @Helper.restful def update_no_slave_purge(self, replica_id, no_slave_purge): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: self.controller.update_no_slave_purge(replica_id, int(no_slave_purge)) self.workers[replica_id].stop() worker = ReplicaWorker(self.persistence, self.controller.get(replica_id), self.config) worker.start() self.workers[replica_id] = worker return "ok" @cherrypy.expose @Helper.restful def purge(self, replica_id): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: self.workers[replica_id].purge() return "ok" @cherrypy.expose @Helper.restful def master_binlogs(self, replica_id): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: return self.workers[replica_id].master_binlogs() @cherrypy.expose @Helper.restful def master_status(self, replica_id): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: return self.workers[replica_id].master_status() @cherrypy.expose @Helper.restful def slave_status(self, replica_id, slave_id): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: return self.workers[replica_id].slave_status(slave_id) @cherrypy.expose @Helper.restful def worker_status(self, replica_id): if not self.workers.has_key(replica_id): raise Exception("worker not exist") else: return self.workers[replica_id].isAlive() @cherrypy.expose @Helper.restful def worker_restart(self, replica_id): if not self.workers.has_key(replica_id): raise Exception("worker not exist") else: self.workers[replica_id].stop() worker = ReplicaWorker(self.persistence, self.controller.get(replica_id)) worker.start() self.workers[replica_id] = worker return "ok"
class ReplicaWorker(threading.Thread): def __init__(self, persistence, dbreplica, config): threading.Thread.__init__(self) self.config = config self.logger = logging.getLogger("cleaner") self.lock = threading.Lock() self.stopped = False self.dbreplica = dbreplica self.dbreplica_controller = DBReplicaController(persistence) self.dbinstance_controller = DBInstanceController(persistence) self._init_replica() self._init_handler() self.monitor = ReplicaMonitor(self.config, self.dbreplica, self.master_handler, self.slaves_handler) def _init_replica(self): master_id = self.dbreplica.master if self.dbreplica.slaves is None or len(self.dbreplica.slaves) == 0: slave_ids = [] else: slave_ids = json.loads(self.dbreplica.slaves) master = self.dbinstance_controller.get(master_id) if master is None: raise Exception("%s no master %s record" % (self.dbreplica.name, master_id)) self.master = master self.slaves = {} for slave_id in slave_ids: slave = self.dbinstance_controller.get(slave_id) if slave is None: raise Exception("%s no slave %s record" % (self.dbreplica.name, slave_id)) self.slaves[slave_id] = slave def _init_handler(self): self.master_handler = ReplicaMasterHandler(self.master) self.slaves_handler = {} for slave_id in self.slaves.keys(): slave = self.slaves[slave_id] self.slaves_handler[slave_id] = ReplicaSlaveHandler(slave) def _earliest_slave_binlog(self): earliest_log_index = -1 earliest_log_name = None for slave_id in self.slaves_handler.keys(): slave_handler = self.slaves_handler[slave_id] (log_index, log_name) = slave_handler.master_binlog() if earliest_log_index < 0 or earliest_log_index > log_index: earliest_log_index = log_index earliest_log_name = log_name return (earliest_log_index, earliest_log_name) def _target_master_binlog(self, earliest_slave_binlog): slave_binlog_index = earliest_slave_binlog[0] master_binlogs = self.master_handler.binlogs_sorted() binlog_length = len(master_binlogs) for i in range(binlog_length): master_binlog_index = master_binlogs[i][0] if slave_binlog_index == master_binlog_index: binlog_window = self.dbreplica.binlog_window if i > binlog_window: return (False, master_binlogs[i - binlog_window], master_binlogs[0], master_binlogs[binlog_length - 1]) else: return (True, None, master_binlogs[0], master_binlogs[binlog_length - 1]) raise Exception( ("%s slave binary log not" + " in master binary logs") % (self.dbreplica.name)) def _do_no_slave_purge(self): master_binlogs = self.master_handler.binlogs_sorted() binlog_window = self.dbreplica.binlog_window binlog_length = len(master_binlogs) if binlog_window + 1 < binlog_length: target_binlog = master_binlogs[binlog_length - binlog_window - 1] self.logger.info( ("%s start no slave purging, " + "earliest_master_binlog %s, " + "target_master_binlog %s, " + "latest_master_binlog %s") % (self.dbreplica.name, master_binlogs[0][1], target_binlog[1], master_binlogs[binlog_length - 1][1])) self.master_handler.purge(target_binlog[1]) else: self.logger.info( ("%s skip no slave purging, " + "earliest_master_binlog %s, " + "latest_master_binlog %s, " + "binlog window %s") % (self.dbreplica.name, master_binlogs[0][1], master_binlogs[binlog_length - 1][1], binlog_window)) def _do_purge(self, no_slave_purge): if len(self.slaves) <= 0 and no_slave_purge == 0: self.logger.info("%s skip purge, no slave" % self.dbreplica.name) elif len(self.slaves) <= 0 and no_slave_purge != 0: self._do_no_slave_purge() else: slave_binlog = self._earliest_slave_binlog() (skip, target_master_binlog, earliest_master_binlog, latest_master_binlog) = self._target_master_binlog(slave_binlog) if not skip: self.logger.info( ("%s start purging, " + "earliest_slave_binlog %s, " + "earliest_master_binlog %s, " + "lateset_master_binlog %s, " + "target_master_binlog %s") % (self.dbreplica.name, slave_binlog[1], earliest_master_binlog[1], latest_master_binlog[1], target_master_binlog[1])) self.master_handler.purge(target_master_binlog[1]) self.logger.info("binary log successfully purged") else: self.logger.info( ("%s skip purge, " + "earliest_slave_binlog %s, " + "earliest_master_binlog %s, " + "latest_master_binlog %s, " + "binlog_window %s") % (self.dbreplica.name, slave_binlog[1], earliest_master_binlog[1], latest_master_binlog[1], self.dbreplica.binlog_window)) def purge(self): if not self.lock.acquire(False): raise Exception("%s another purge is running" % (self.dbreplica.name)) else: try: self._do_purge(1) self.lock.release() except Exception as e: self.lock.release() raise e def master_binlogs(self): return self.master_handler.binlogs() def master_status(self): return self.master_handler.status() def slave_status(self, slave_id): if self.slaves.has_key(slave_id): return self.slaves_handler[slave_id].status() else: raise Exception("%s no such slave" % self.dbreplica.name) def stop(self): self.lock.acquire() self.stopped = True self.monitor.stop() self.lock.release() def run(self): self.logger.info("worker %s started" % self.dbreplica.name) self.monitor.start() while True: time.sleep(self.dbreplica.check_period) if not self.stopped: self.lock.acquire() try: self._do_purge(self.dbreplica.no_slave_purge) self.lock.release() except Exception as e: self.lock.release() self.monitor.send_mail("purge error " + str(e), traceback.format_exc()) self.logger.error("%s run purge error: %s" % (self.dbreplica.name, str(e))) else: self.logger.info("worker %s stopped" % self.dbreplica.name) break if not self.monitor.isstopped(): self.monitor.stop()
class DBReplicaRestful(): def __init__(self, config, persistence): self.config = config self.persistence = persistence self.controller = DBReplicaController(persistence) self.dbinstance_controller = DBInstanceController(persistence) self._init_worker() def _init_worker(self): self.workers = {} dbreplicas = self.controller.get_all() for dbreplica in dbreplicas: worker = ReplicaWorker(self.persistence, dbreplica, self.config) worker.start() self.workers[dbreplica.id] = worker @cherrypy.expose @Helper.restful def add(self, replica_id, name, master, slaves, check_period=60, binlog_window=0, no_slave_purge=1): if self.workers.has_key(replica_id): raise Exception("duplicate replication") else: dbreplica = DBReplica(replica_id, name, master, slaves, int(check_period), int(binlog_window), int(no_slave_purge)) worker = ReplicaWorker(self.persistence, dbreplica, self.config) worker.start() self.workers[replica_id] = worker self.controller.add(dbreplica) return "ok" @cherrypy.expose @Helper.restful def delete(self, replica_id): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: worker = self.workers.pop(replica_id) worker.stop() self.controller.delete(replica_id) return "ok" @cherrypy.expose @Helper.restful def add_slave(self, replica_id, slave_id): if not self.workers.has_key(replica_id): raise Exception("replication not exist") elif self.dbinstance_controller.get(slave_id) is None: raise Exception("no %s slave instance" % slave_id) else: self.controller.add_slave(replica_id, slave_id) self.workers[replica_id].stop() worker = ReplicaWorker(self.persistence, self.controller.get(replica_id), self.config) worker.start() self.workers[replica_id] = worker return "ok" @cherrypy.expose @Helper.restful def add_slaves(self, replica_id, slave_ids): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: slaves = json.loads(slave_ids) for slave_id in slaves: if self.dbinstance_controller.get(slave_id) is None: raise Exception("no %s slave instance" % slave_id) self.controller.add_slaves(replica_id, slaves) self.workers[replica_id].stop() worker = ReplicaWorker(self.persistence, self.controller.get(replica_id), self.config) worker.start() self.workers[replica_id] = worker return "ok" @cherrypy.expose @Helper.restful def del_slave(self, replica_id, slave_id): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: self.controller.del_slave(replica_id, slave_id) self.workers[replica_id].stop() worker = ReplicaWorker(self.persistence, self.controller.get(replica_id), self.config) worker.start() self.workers[replica_id] = worker return "ok" @cherrypy.expose @Helper.restful def update_check_period(self, replica_id, check_period): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: self.controller.update_check_period(replica_id, int(check_period)) self.workers[replica_id].stop() worker = ReplicaWorker(self.persistence, self.controller.get(replica_id), self.config) worker.start() self.workers[replica_id] = worker return "ok" @cherrypy.expose @Helper.restful def update_binlog_window(self, replica_id, binlog_window): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: self.controller.update_binlog_window(replica_id, int(binlog_window)) self.workers[replica_id].stop() worker = ReplicaWorker(self.persistence, self.controller.get(replica_id), self.config) worker.start() self.workers[replica_id] = worker return "ok" @cherrypy.expose @Helper.restful def update_no_slave_purge(self, replica_id, no_slave_purge): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: self.controller.update_no_slave_purge(replica_id, int(no_slave_purge)) self.workers[replica_id].stop() worker = ReplicaWorker(self.persistence, self.controller.get(replica_id), self.config) worker.start() self.workers[replica_id] = worker return "ok" @cherrypy.expose @Helper.restful def purge(self, replica_id): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: self.workers[replica_id].purge() return "ok" @cherrypy.expose @Helper.restful def master_binlogs(self, replica_id): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: return self.workers[replica_id].master_binlogs() @cherrypy.expose @Helper.restful def master_status(self, replica_id): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: return self.workers[replica_id].master_status() @cherrypy.expose @Helper.restful def slave_status(self, replica_id, slave_id): if not self.workers.has_key(replica_id): raise Exception("replication not exist") else: return self.workers[replica_id].slave_status(slave_id) @cherrypy.expose @Helper.restful def worker_status(self, replica_id): if not self.workers.has_key(replica_id): raise Exception("worker not exist") else: return self.workers[replica_id].isAlive() @cherrypy.expose @Helper.restful def worker_restart(self, replica_id): if not self.workers.has_key(replica_id): raise Exception("worker not exist") else: self.workers[replica_id].stop() worker = ReplicaWorker(self.persistence, self.controller.get(replica_id)) worker.start() self.workers[replica_id] = worker return "ok"
class ReplicaWorker(threading.Thread): def __init__(self, persistence, dbreplica, config): threading.Thread.__init__(self) self.config = config self.logger = logging.getLogger("cleaner") self.lock = threading.Lock() self.stopped = False self.dbreplica = dbreplica self.dbreplica_controller = DBReplicaController(persistence) self.dbinstance_controller = DBInstanceController(persistence) self._init_replica() self._init_handler() self.monitor = ReplicaMonitor(self.config, self.dbreplica, self.master_handler, self.slaves_handler) def _init_replica(self): master_id = self.dbreplica.master if self.dbreplica.slaves is None or len(self.dbreplica.slaves) == 0: slave_ids = [] else: slave_ids = json.loads(self.dbreplica.slaves) master = self.dbinstance_controller.get(master_id) if master is None: raise Exception("%s no master %s record" % (self.dbreplica.name, master_id)) self.master = master self.slaves = {} for slave_id in slave_ids: slave = self.dbinstance_controller.get(slave_id) if slave is None: raise Exception("%s no slave %s record" % (self.dbreplica.name, slave_id)) self.slaves[slave_id] = slave def _init_handler(self): self.master_handler = ReplicaMasterHandler(self.master) self.slaves_handler = {} for slave_id in self.slaves.keys(): slave = self.slaves[slave_id] self.slaves_handler[slave_id] = ReplicaSlaveHandler(slave) def _earliest_slave_binlog(self): earliest_log_index = -1 earliest_log_name = None for slave_id in self.slaves_handler.keys(): slave_handler = self.slaves_handler[slave_id] (log_index, log_name) = slave_handler.master_binlog() if earliest_log_index < 0 or earliest_log_index > log_index: earliest_log_index = log_index earliest_log_name = log_name return (earliest_log_index, earliest_log_name) def _target_master_binlog(self, earliest_slave_binlog): slave_binlog_index = earliest_slave_binlog[0] master_binlogs = self.master_handler.binlogs_sorted() binlog_length = len(master_binlogs) for i in range(binlog_length): master_binlog_index = master_binlogs[i][0] if slave_binlog_index == master_binlog_index: binlog_window = self.dbreplica.binlog_window if i > binlog_window: return (False, master_binlogs[i - binlog_window], master_binlogs[0], master_binlogs[binlog_length - 1]) else: return (True, None, master_binlogs[0], master_binlogs[binlog_length - 1]) raise Exception(("%s slave binary log not" + " in master binary logs") % (self.dbreplica.name)) def _do_no_slave_purge(self): master_binlogs = self.master_handler.binlogs_sorted() binlog_window = self.dbreplica.binlog_window binlog_length = len(master_binlogs) if binlog_window + 1 < binlog_length: target_binlog = master_binlogs[binlog_length-binlog_window-1] self.logger.info(("%s start no slave purging, " + "earliest_master_binlog %s, " + "target_master_binlog %s, " + "latest_master_binlog %s") % (self.dbreplica.name, master_binlogs[0][1], target_binlog[1], master_binlogs[binlog_length-1][1])) self.master_handler.purge(target_binlog[1]) else: self.logger.info(("%s skip no slave purging, " + "earliest_master_binlog %s, " + "latest_master_binlog %s, " + "binlog window %s") % (self.dbreplica.name, master_binlogs[0][1], master_binlogs[binlog_length-1][1], binlog_window)) def _do_purge(self, no_slave_purge): if len(self.slaves) <= 0 and no_slave_purge == 0: self.logger.info("%s skip purge, no slave" % self.dbreplica.name) elif len(self.slaves) <= 0 and no_slave_purge != 0: self._do_no_slave_purge() else: slave_binlog = self._earliest_slave_binlog() (skip, target_master_binlog, earliest_master_binlog, latest_master_binlog) = self._target_master_binlog(slave_binlog) if not skip: self.logger.info(("%s start purging, " + "earliest_slave_binlog %s, " + "earliest_master_binlog %s, " + "lateset_master_binlog %s, " + "target_master_binlog %s") % (self.dbreplica.name, slave_binlog[1], earliest_master_binlog[1], latest_master_binlog[1], target_master_binlog[1])) self.master_handler.purge(target_master_binlog[1]) self.logger.info("binary log successfully purged") else: self.logger.info(("%s skip purge, "+ "earliest_slave_binlog %s, " + "earliest_master_binlog %s, " + "latest_master_binlog %s, " + "binlog_window %s") % (self.dbreplica.name, slave_binlog[1], earliest_master_binlog[1], latest_master_binlog[1], self.dbreplica.binlog_window)) def purge(self): if not self.lock.acquire(False): raise Exception("%s another purge is running" % (self.dbreplica.name)) else: try: self._do_purge(1) self.lock.release() except Exception as e: self.lock.release() raise e def master_binlogs(self): return self.master_handler.binlogs() def master_status(self): return self.master_handler.status() def slave_status(self, slave_id): if self.slaves.has_key(slave_id): return self.slaves_handler[slave_id].status() else: raise Exception("%s no such slave" % self.dbreplica.name) def stop(self): self.lock.acquire() self.stopped = True self.monitor.stop() self.lock.release() def run(self): self.logger.info("worker %s started" % self.dbreplica.name) self.monitor.start() while True: time.sleep(self.dbreplica.check_period) if not self.stopped: self.lock.acquire() try: self._do_purge(self.dbreplica.no_slave_purge) self.lock.release() except Exception as e: self.lock.release() self.monitor.send_mail("purge error " + str(e), traceback.format_exc()) self.logger.error("%s run purge error: %s" % (self.dbreplica.name, str(e))) else: self.logger.info("worker %s stopped" % self.dbreplica.name) break if not self.monitor.isstopped(): self.monitor.stop()