def max_timeLimit(self, connection=None): if connection is None: connection = self.connection # Work out which appserver we're running against # 1. It's usually 'App-Services' marklogic = MarkLogic(connection) server = marklogic.http_server('App-Services') if server.exists() and server.port() == connection.port: pass else: slist = marklogic.http_servers() server = None while slist and server is None: group, name = re.split("\\|", slist.pop()) try: check = marklogic.http_server(name, group=group) if check.port() == connection.port: server = check except UnexpectedManagementAPIResponse: pass except: raise if (server is not None and server.exists() \ and server.port() == connection.port): return server.max_time_limit() else: # Oh, heck, just go with a default return 600 sys.exit(1)
def stop(self, args, config, connection): status = self.status(args, config, connection, internal=True) if status == 'down': print("MarkLogic server on {0} appears to be {1}.".format( connection.host, status)) else: if 'cluster' in args: hostname = connection.host if hostname == 'localhost': hostname = socket.gethostname() cluster = LocalCluster(connection=connection).read() print("Shutting down cluster...") cluster.shutdown() else: hostname = connection.host if hostname == 'localhost': hostname = socket.gethostname() host = Host(hostname, connection=connection).read() if host.group_name() is None: ml = MarkLogic(connection) if len(ml.hosts()) == 1: host = Host(ml.hosts()[0], connection=connection).read() else: print("Failed to identify host:", ml.hosts()) sys.exit(1) print("Shutting down host: " + host.host_name()) host.shutdown() status = self.status(args, config, connection, internal=True) while status == 'up': time.sleep(2) status = self.status(args, config, connection, internal=True)
def stop(self, args, config, connection): status = self.status(args, config, connection, internal=True) if status == 'down': print("MarkLogic server on {0} appears to be {1}." .format(connection.host, status)) else: if 'cluster' in args: hostname = connection.host if hostname == 'localhost': hostname = socket.gethostname() cluster = LocalCluster(connection=connection).read() print("Shutting down cluster...") cluster.shutdown() else: hostname = connection.host if hostname == 'localhost': hostname = socket.gethostname() host = Host(hostname,connection=connection).read() if host.group_name() is None: ml = MarkLogic(connection) if len(ml.hosts()) == 1: host = Host(ml.hosts()[0], connection=connection).read() else: print("Failed to identify host:",ml.hosts()) sys.exit(1) print("Shutting down host: " + host.host_name()) host.shutdown() status = self.status(args,config,connection,internal=True) while status == 'up': time.sleep(2) status = self.status(args,config,connection,internal=True)
def join(self, args, config, connection): cluster = LocalCluster(connection=connection) cluster.read() self.logger.info("Initializing {0}...".format(args['host'])) MarkLogic.instance_init(args['host']) host = Host(args['host']) self.logger.info("Joining cluster...") cluster.add_host(host)
def test_create_database(self): self.marklogic = MarkLogic(self.connection) dbname = 'test-mma-database' mma = MMA(self.connection) mma.run(['create', 'database', dbname]) assert dbname in self.marklogic.databases() mma.run(['delete', 'database', dbname]) assert dbname not in self.marklogic.databases()
class TestMmaDatabase(MLConfig): def test_create_database(self): self.marklogic = MarkLogic(self.connection) dbname = 'test-mma-database' mma = MMA(self.connection) mma.run(['create', 'database', dbname]) assert dbname in self.marklogic.databases() mma.run(['delete', 'database', dbname]) assert dbname not in self.marklogic.databases()
def couple_cluster(self): conn = Connection(self.host, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) if self.couple is not None: for couple in self.couple: print("{0}: couple with {1}...".format(self.host,couple)) altconn = Connection(couple, HTTPDigestAuth(self.couple_user, self.couple_pass)) altml = MarkLogic(altconn) altcluster = altml.cluster() cluster = self.marklogic.cluster() cluster.couple(altcluster) print("Finished")
def couple_cluster(self): conn = Connection(self.host, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) if self.couple is not None: for couple in self.couple: print("{0}: couple with {1}...".format(self.host, couple)) altconn = Connection( couple, HTTPDigestAuth(self.couple_user, self.couple_pass)) altml = MarkLogic(altconn) altcluster = altml.cluster() cluster = self.marklogic.cluster() cluster.couple(altcluster) print("Finished")
def build_cluster(host, adminuser, adminpass, couple_user, couple_pass, cluster_host_list): conn = Connection(host, HTTPDigestAuth(adminuser, adminpass)) marklogic = MarkLogic(conn) if cluster_host_list is not None: for couple in cluster_host_list: print("{0}: couple with {1}...".format(host, couple)) altconn = Connection(couple, HTTPDigestAuth(couple_user, couple_pass)) altml = MarkLogic(altconn) altcluster = altml.cluster() cluster = marklogic.cluster() cluster.couple(altcluster) print("Finished")
def init(self, args, config, connection): status = self.status(args,config,connection,internal=True) if status != 'up': try: data = config[args['config']]['datadir'] print("Clearing {0}...".format(data)) shutil.rmtree(data) os.mkdir(data) except KeyError: pass self.start(args,config,connection) print("Initializing {0}...".format(connection.host)) MarkLogic.instance_init(connection.host) MarkLogic.instance_admin(connection.host, args['realm'], connection.auth.username, connection.auth.password)
def init(self, args, config, connection): status = self.status(args, config, connection, internal=True) if status != 'up': try: data = config[args['config']]['datadir'] print("Clearing {0}...".format(data)) shutil.rmtree(data) os.mkdir(data) except KeyError: pass self.start(args, config, connection) print("Initializing {0}...".format(connection.host)) MarkLogic.instance_init(connection.host) MarkLogic.instance_admin(connection.host, args['realm'], connection.auth.username, connection.auth.password)
def create_forests(self): conn = Connection(self.host, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) if self.failhost is None and self.failover != "none": print("Invalid configuration, specify failover-host for failover:", self.failover) sys.exit(1) if self.hosts is None: host_names = self.marklogic.hosts() else: host_names = self.hosts exists = [] host_index = 0 for host_name in host_names: host_index += 1 for count in range(1,self.forests + 1): name = self.prefix + "-" + str(host_index) + "-" + str(count) forest = Forest(name, host_name, conn) if forest.exists(): exists.append(host_name + ":" + name) if len(exists) != 0: print("Abort: forest(s) already exist:") for f in exists: print(" ", f) sys.exit(1) host_index = 0 for host_name in host_names: host_index += 1 for count in range(self.start,self.start + self.forests): name = self.prefix + "-" + str(host_index) + "-" + str(count) forest = Forest(name, host_name, conn) if self.data_dir is not None: forest.set_data_directory(self.data_dir) if self.large_data_dir is not None: forest.set_large_data_directory(self.large_data_dir) if self.fast_data_dir is not None: forest.set_fast_data_directory(self.fast_data_dir) if self.failhost is not None: forest.set_failover(self.failover) forest.set_failover_host_names(self.failhost) if self.database is not None: forest.set_database(self.database) print("Create forest " + name + " on " + host_name) if self.dry_run: print(json.dumps(forest.marshal(), sort_keys=True, indent=2)) else: forest.create() print("Finished")
def ml_init(self, hostname): print("{0}: initialize host...".format(hostname)) try: host = MarkLogic.instance_init(hostname) except UnauthorizedAPIRequest: # Assume that this happened because the host is already initialized host = Host(hostname) return host.just_initialized()
class TestMmaDatabase(unittest.TestCase): """ Basic creation test function. """ def __init__(self, argv): super().__init__(argv) self.connection = Connection(tc.hostname, HTTPDigestAuth(tc.admin, tc.password)) self.marklogic = MarkLogic(self.connection) def test_create_database(self): dbname = 'test-mma-database' mma = MMA(self.connection) mma.run(['create','database',dbname]) self.assertTrue(dbname in self.marklogic.databases()) mma.run(['delete','database',dbname]) self.assertFalse(dbname in self.marklogic.databases())
def debug(self, args, config, connection): section = config[args['config']] if 'diagnostic-events' in section: events = [] devents = section['diagnostic-events'].strip() if devents != '': for event in devents.split(","): events.append(event.strip()) else: events = [] ml = MarkLogic(connection) group = ml.group(args['group']) group.set_events(events) if len(events) == 0: group.set_events_activated(False) else: group.set_events_activated(True) group.update()
def init(self, args, config, connection): status = self.status(args,config,connection,internal=True) if status == 'up': print("Cannot initialize a running host!") sys.exit(1) else: if connection.host == 'localhost': try: data = config[args['config']]['datadir'] print("Clearing {0}...".format(data)) self._clear_directory(data) except KeyError: pass else: self.logger.info("Skipping clear data directory; not localhost") self.start(args,config,connection) print("Initializing {0}...".format(connection.host)) MarkLogic.instance_init(connection.host) MarkLogic.instance_admin(connection.host, args['realm'], connection.auth.username, connection.auth.password)
def debug(self, args, config, connection): section = config[args['config']] if 'diagnostic-events' in section: events = section['diagnostic-events'].strip() if events == '': events = [] else: events = re.sub(r'\s+', '', events) events = events.split(',') else: events = [] ml = MarkLogic(connection) group = ml.group(args['group']) group.set_events(events) if len(events) == 0: group.set_events_activated(False) else: group.set_events_activated(True) group.update()
def init(self, args, config, connection): status = self.status(args, config, connection, internal=True) if status == 'up': print("Cannot initialize a running host!") sys.exit(1) else: if connection.host == 'localhost': try: data = config[args['config']]['datadir'] print("Clearing {0}...".format(data)) self._clear_directory(data) except KeyError: pass else: self.logger.info( "Skipping clear data directory; not localhost") self.start(args, config, connection) print("Initializing {0}...".format(connection.host)) MarkLogic.instance_init(connection.host) MarkLogic.instance_admin(connection.host, args['realm'], connection.auth.username, connection.auth.password)
def setup_cluster(self): if self.couple is not None and self.couple_pass is None: self.couple_pass = self.adminpass self.couple_user = self.adminuser if self.boothost is None: self.boothost = self.host[0] self.host.remove(self.boothost) if self.ml_init(self.boothost): self.ml_security(self.boothost) conn = Connection(self.boothost, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) for hostname in self.host: self.ml_init(hostname) self.ml_join(self.boothost, hostname) if self.name is not None: print("{0}: rename cluster...".format(self.boothost)) cluster = self.marklogic.cluster() cluster.set_cluster_name(self.name) cluster.update() if self.couple is not None: for couple in self.couple: print("{0}: couple with {1}...".format(self.boothost, couple)) altconn = Connection(couple, HTTPDigestAuth(self.couple_user, self.couple_pass)) altml = MarkLogic(altconn) altcluster = altml.cluster() cluster = self.marklogic.cluster() cluster.couple(altcluster) print("Finished")
def setup_cluster(self): if self.couple is not None and self.couple_pass is None: self.couple_pass = self.adminpass self.couple_user = self.adminuser if self.boothost is None: self.boothost = self.host[0] self.host.remove(self.boothost) if self.ml_init(self.boothost): self.ml_security(self.boothost) conn = Connection(self.boothost, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) for hostname in self.host: self.ml_init(hostname) self.ml_join(self.boothost, hostname) if self.name is not None: print("{0}: rename cluster...".format(self.boothost)) cluster = self.marklogic.cluster() cluster.set_cluster_name(self.name) cluster.update() if self.couple is not None: for couple in self.couple: print("{0}: couple with {1}...".format(self.boothost, couple)) altconn = Connection( couple, HTTPDigestAuth(self.couple_user, self.couple_pass)) altml = MarkLogic(altconn) altcluster = altml.cluster() cluster = self.marklogic.cluster() cluster.couple(altcluster) print("Finished")
def ml_init(self, ip, container): if container in self.hostname: hostname = self.hostname[container] else: hostname = container print("{0}: initialize host {1}...".format(ip,hostname)) try: host = MarkLogic.instance_init(ip) except UnauthorizedAPIRequest: # Assume that this happened because the host is already initialized host = Host(ip) self.blacklist[container] = "used" self.save_blacklist() return host.just_initialized()
def ml_init(self, ip, container): if container in self.hostname: hostname = self.hostname[container] else: hostname = container print("{0}: initialize host {1}...".format(ip, hostname)) try: host = MarkLogic.instance_init(ip) except UnauthorizedAPIRequest: # Assume that this happened because the host is already initialized host = Host(ip) self.blacklist[container] = "used" self.save_blacklist() return host.just_initialized()
def connect(self, args): try: adminuser, adminpass = re.split(":", args['credentials']) except ValueError: print("--credentials value must be 'user:password':"******"requests").setLevel(logging.WARNING) logging.getLogger("marklogic").setLevel(logging.DEBUG) self.connection \ = Connection(args['hostname'], HTTPDigestAuth(adminuser, adminpass)) self.mls = MarkLogic(self.connection) self.args = args
class Couple: def __init__(self): self.marklogic = None self.adminuser = "******" self.adminpass = "******" self.host = "localhost" self.couple = None self.couple_user = None self.couple_pass = None def set_credentials(self, user, password): self.adminuser = user self.adminpass = password def set_host(self, host): self.host = host def set_couple(self, couple): self.couple = couple def set_couple_credentials(self, user, password): self.couple_user = user self.couple_pass = password def couple_cluster(self): conn = Connection(self.host, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) if self.couple is not None: for couple in self.couple: print("{0}: couple with {1}...".format(self.host, couple)) altconn = Connection( couple, HTTPDigestAuth(self.couple_user, self.couple_pass)) altml = MarkLogic(altconn) altcluster = altml.cluster() cluster = self.marklogic.cluster() cluster.couple(altcluster) print("Finished")
class Couple: def __init__(self): self.marklogic = None self.adminuser="******" self.adminpass="******" self.host="localhost" self.couple=None self.couple_user=None self.couple_pass=None def set_credentials(self,user,password): self.adminuser = user self.adminpass = password def set_host(self,host): self.host = host def set_couple(self,couple): self.couple = couple def set_couple_credentials(self,user,password): self.couple_user = user self.couple_pass = password def couple_cluster(self): conn = Connection(self.host, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) if self.couple is not None: for couple in self.couple: print("{0}: couple with {1}...".format(self.host,couple)) altconn = Connection(couple, HTTPDigestAuth(self.couple_user, self.couple_pass)) altml = MarkLogic(altconn) altcluster = altml.cluster() cluster = self.marklogic.cluster() cluster.couple(altcluster) print("Finished")
class CreateForests: def __init__(self, marklogic=None, host="localhost", hosts=None, prefix="F", forests=2, start_forest=1, failover="none", failover_host=None, adminuser="******", adminpass="******", data_dir=None, large_data_dir=None, fast_data_dir=None, database=None, dry_run=False): self.marklogic = marklogic self.host = host self.hosts = hosts if isinstance(hosts, list) or None else [hosts] self.prefix = prefix self.forests = forests self.start = start_forest self.failover = failover self.failhost = failover_host self.adminuser = adminuser self.adminpass = adminpass self.data_dir = data_dir self.large_data_dir = large_data_dir self.fast_data_dir = fast_data_dir self.database = database self.dry_run = dry_run def create_forests(self): conn = Connection(self.host, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) if self.failhost is None and self.failover != "none": print("Invalid configuration, specify failover-host for failover:", self.failover) sys.exit(1) if self.hosts is None: host_names = self.marklogic.hosts() else: host_names = self.hosts exists = [] for host_index, host_name in enumerate(host_names): for count in range(1, self.forests + 1): name = self.prefix + "-" + str(host_index) + "-" + str(count) forest = Forest(name, host_name, conn) if forest.exists(): exists.append(host_name + ":" + name) if len(exists) != 0: print("Abort: forest(s) already exist:") for f in exists: print(" ", f) sys.exit(1) for host_index, host_name in enumerate(host_names): for count in range(self.start, self.start + self.forests): name = self.prefix + "-" + str(host_index) + "-" + str(count) forest = Forest(name, host_name, conn) if self.data_dir is not None: forest.set_data_directory(self.data_dir) if self.large_data_dir is not None: forest.set_large_data_directory(self.large_data_dir) if self.fast_data_dir is not None: forest.set_fast_data_directory(self.fast_data_dir) if self.failhost is not None: forest.set_failover(self.failover) forest.set_failover_host_names(self.failhost) if self.database is not None: forest.set_database(self.database) print("Create forest " + name + " on " + host_name) if self.dry_run: print(json.dumps(forest.marshal(), sort_keys=True, indent=2)) else: forest.create() print("Finished")
def setup_cluster(self): if self.couple is not None and self.couple_pass is None: self.couple_pass = self.adminpass self.couple_user = self.adminuser self.load_blacklist() self.find_containers() if not self.container_list: print("There must be at least one unused container running.") sys.exit(1) if self.localimage: ps = os.popen("docker exec " + self.container_list[0] + " ip route") for line in ps.readlines(): match = re.match("^default via (\S+)", line) if match: self.localip = match.group(1) if self.localip is None: print("Cannot find IP address of localhost!?") sys.exit(1) # Find the bootstrap image if self.localimage: pass else: self.bootimage = self.pick_image(self.bootimage) self.container_list.remove(self.bootimage) self.cluster_list = self.pick_containers() self.display_info() #sys.exit(1) # Initialize the bootstrap image, if necessary if self.localimage: bootip = self.localip else: bootip = self.ipaddr[self.bootimage] if self.ml_init(bootip, self.bootimage): self.ml_security(bootip, self.bootimage) conn = Connection(bootip, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) for container in self.cluster_list: if container == self.bootimage: continue ip = self.ipaddr[container] self.ml_init(ip, container) self.ml_join(bootip, ip) if self.name is not None: print("{0}: rename cluster...".format(bootip)) cluster = self.marklogic.cluster() cluster.set_cluster_name(self.name) cluster.update() if self.couple is not None: for couple in self.couple: print("{0}: couple with {1}...".format(bootip,couple)) altconn = Connection(couple, HTTPDigestAuth(self.couple_user, self.couple_pass)) altml = MarkLogic(altconn) altcluster = altml.cluster() cluster = self.marklogic.cluster() cluster.couple(altcluster) print("Finished")
class App: def __init__(self): self.marklogic = None self.uname = pwd.getpwuid(os.getuid()).pw_name self.adminuser = "******" self.adminpass = "******" self.realm = "public" self.host = None self.boothost = None self.couple = None self.couple_user = None self.couple_pass = None self.name = None def set_credentials(self, user, password): self.adminuser = user self.adminpass = password def set_realm(self, realm): self.realm = realm def set_boot_host(self, host): self.boothost = host def set_host(self, name): self.host = name def set_couple(self, couple): self.couple = couple def set_couple_credentials(self, user, password): self.couple_user = user self.couple_pass = password def set_name(self, name): self.name = name def setup_cluster(self): if self.couple is not None and self.couple_pass is None: self.couple_pass = self.adminpass self.couple_user = self.adminuser if self.boothost is None: self.boothost = self.host[0] self.host.remove(self.boothost) if self.ml_init(self.boothost): self.ml_security(self.boothost) conn = Connection(self.boothost, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) for hostname in self.host: self.ml_init(hostname) self.ml_join(self.boothost, hostname) if self.name is not None: print("{0}: rename cluster...".format(self.boothost)) cluster = self.marklogic.cluster() cluster.set_cluster_name(self.name) cluster.update() if self.couple is not None: for couple in self.couple: print("{0}: couple with {1}...".format(self.boothost, couple)) altconn = Connection( couple, HTTPDigestAuth(self.couple_user, self.couple_pass)) altml = MarkLogic(altconn) altcluster = altml.cluster() cluster = self.marklogic.cluster() cluster.couple(altcluster) print("Finished") def ml_init(self, hostname): print("{0}: initialize host...".format(hostname)) try: host = MarkLogic.instance_init(hostname) except UnauthorizedAPIRequest: # Assume that this happened because the host is already initialized host = Host(hostname) return host.just_initialized() def ml_join(self, boothost, hostname): print("{0}: join cluster with {1}...".format(hostname, boothost)) cluster = self.marklogic.cluster() host = Host(hostname) cluster.add_host(host) def ml_security(self, hostname): print("{0}: initialize security...".format(hostname)) MarkLogic.instance_admin(hostname, self.realm, self.adminuser, self.adminpass)
class Docker: def __init__(self): self.marklogic = None self.uname = pwd.getpwuid(os.getuid()).pw_name self.adminuser = "******" self.adminpass = "******" self.realm = "public" self.imatch = "ml[0-9]" self.bootimage = None self.localimage = False self.count = None self.couple = None self.couple_user = None self.couple_pass = None self.name = None self.blacklist_file = "/tmp/{0}.docker.skip".format(self.uname) self.blacklist = {} self.container_list = [] self.cluster_list = [] self.containers = {} self.localip = None self.ipaddr = {} self.hostname = {} def set_credentials(self, user, password): self.adminuser = user self.adminpass = password def set_realm(self, realm): self.realm = realm def set_image_match(self, match): self.imatch = match def set_boot_image(self, image): self.bootimage = image self.localimage = (image == "localhost") def set_count(self, count): self.count = count def set_couple(self, couple): self.couple = couple def set_couple_credentials(self, user, password): self.couple_user = user self.couple_pass = password def set_name(self, name): self.name = name def setup_cluster(self): if self.couple is not None and self.couple_pass is None: self.couple_pass = self.adminpass self.couple_user = self.adminuser self.load_blacklist() self.find_containers() if not self.container_list: print("There must be at least one unused container running.") sys.exit(1) if self.localimage: ps = os.popen("docker exec " + self.container_list[0] + " ip route") for line in ps.readlines(): match = re.match("^default via (\S+)", line) if match: self.localip = match.group(1) if self.localip is None: print("Cannot find IP address of localhost!?") sys.exit(1) # Find the bootstrap image if self.localimage: pass else: self.bootimage = self.pick_image(self.bootimage) if self.bootimage in self.container_list: self.container_list.remove(self.bootimage) self.cluster_list = self.pick_containers() self.display_info() #sys.exit(1) # Initialize the bootstrap image, if necessary if self.localimage: bootip = self.localip else: bootip = self.ipaddr[self.bootimage] if self.ml_init(bootip, self.bootimage): self.ml_security(bootip, self.bootimage) conn = Connection(bootip, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) for container in self.cluster_list: if container == self.bootimage: continue ip = self.ipaddr[container] self.ml_init(ip, container) self.ml_join(bootip, ip) if self.name is not None: print("{0}: rename cluster...".format(bootip)) cluster = self.marklogic.cluster() cluster.set_cluster_name(self.name) cluster.update() if self.couple is not None: for couple in self.couple: print("{0}: couple with {1}...".format(bootip, couple)) altconn = Connection( couple, HTTPDigestAuth(self.couple_user, self.couple_pass)) altml = MarkLogic(altconn) altcluster = altml.cluster() cluster = self.marklogic.cluster() cluster.couple(altcluster) print("Finished") def ml_init(self, ip, container): if container in self.hostname: hostname = self.hostname[container] else: hostname = container print("{0}: initialize host {1}...".format(ip, hostname)) try: host = MarkLogic.instance_init(ip) except UnauthorizedAPIRequest: # Assume that this happened because the host is already initialized host = Host(ip) self.blacklist[container] = "used" self.save_blacklist() return host.just_initialized() def ml_join(self, bootip, ip): print("{0}: join cluster with {1}...".format(ip, bootip)) cluster = self.marklogic.cluster() host = Host(ip) cluster.add_host(host) def ml_security(self, ip, container): print("{0}: initialize security...".format(ip)) MarkLogic.instance_admin(ip, self.realm, self.adminuser, self.adminpass) self.blacklist[container] = "boot" self.save_blacklist() def pick_image(self, name): if name is None: for container in self.container_list: if not container in self.blacklist: return container print("Cannot find unused container") sys.exit(1) for key in self.hostname: if name == self.hostname[key]: return key count = 0 container = None for key in self.ipaddr: if name == self.ipaddr[key]: return key if re.match("^" + name, key): count += 1 container = key if count == 1: return container if count > 1: print("Ambiguous container id:", name) sys.exit(1) for key in self.containers: image = self.containers[key] if (key == self.bootimage or re.match(self.bootimage, image)): return key print("Cannot find container \"{0}\".".format(name)) sys.exit(1) def pick_containers(self): if self.count is None: count = -1 else: count = self.count - 1 cset = set() if not self.localimage: cset.add(self.bootimage) for container in self.container_list: if container == self.bootimage or container in self.blacklist: continue if count == 0: continue cset.add(container) count -= 1 containers = [] for key in cset: containers.append(key) return containers def load_blacklist(self): try: f = open(self.blacklist_file, "r") for line in f.readlines(): (container, flag) = line.strip().split(" ") self.blacklist[container] = flag f.close() except FileNotFoundError: pass def save_blacklist(self): f = open(self.blacklist_file, "w") for key in self.blacklist: if self.blacklist[key] != "gone": f.write("{0} {1}\n".format(key, self.blacklist[key])) f.close() def find_containers(self): self.load_blacklist() ps = os.popen("docker ps") for line in ps.readlines(): match = re.match("([0-9a-f]+)\s+(\S+)", line) if match: container = match.group(1) image = match.group(2) ipx = os.popen("docker inspect {0}".format(container)) ip = None hostname = None for line in ipx.readlines(): match = re.match('\s+"IPAddress": "([^"]+)"', line) if match: ip = match.group(1) match = re.match('\s+"Hostname": "([^"]+)"', line) if match: hostname = match.group(1) if ip is None: print("Cannot read IP address of container {0}".format( container)) else: self.ipaddr[container] = ip if hostname is None: print("Cannot read hostname of container {0}".format( container)) else: self.hostname[container] = hostname if re.match(self.imatch, image): self.container_list = [container] + self.container_list self.containers[container] = image for container in self.blacklist: if not container in self.containers: self.blacklist[container] = "gone" def display_info(self): # Display information about docker containers # Sigh. Report formatting is such a drag... dinfo = [] ps = os.popen("docker ps") for line in ps.readlines(): match = re.match("([0-9a-f]+)\s+(\S+)", line) if match: container = match.group(1) image = match.group(2) ip = self.ipaddr[container] hostname = self.hostname[container] attr = [] if container in self.blacklist: attr.append("used") if self.blacklist[container] == "boot": attr.append("boot") else: if not container in self.cluster_list: attr.append("skip") else: if re.match(self.imatch, image): attr.append("*") if container == self.bootimage: attr.append("boot") else: attr.append("!~" + self.imatch) data = { "container": container, "image": image, "ipaddr": ip, "hostname": hostname, "flags": ", ".join(attr) } dinfo = [data] + dinfo headers = { "container": "Container", "image": "Image", "hostname": "Hostname", "ipaddr": "IP Addr", "flags": "Flags" } maxlen = {} for key in dinfo[0]: flen = len(headers[key]) for data in dinfo: if len(data[key]) > flen: flen = len(data[key]) maxlen[key] = flen fstr = "%-{0}s %-{1}s %-{2}s %-{3}s %s".format( maxlen['container'], maxlen['image'], maxlen['hostname'], maxlen['ipaddr']) print(fstr % (headers['container'], headers['image'], headers['hostname'], headers['ipaddr'], headers['flags'])) for data in dinfo: print(fstr % (data['container'], data['image'], data['hostname'], data['ipaddr'], data['flags']))
def ml_security(self, hostname): print("{0}: initialize security...".format(hostname)) MarkLogic.instance_admin(hostname, self.realm, self.adminuser, self.adminpass)
class CreateForests: def __init__(self): self.marklogic = None self.host = "localhost" self.hosts = None self.prefix = "F" self.forests = 2 self.start = 1 self.failover = "none" self.failhost = None self.adminuser = "******" self.adminpass = "******" self.data_dir = None self.large_data_dir = None self.fast_data_dir = None self.database = None self.dry_run = False # TODO: better checking of argument types def set_rest_host(self, host): self.host = host return self def set_hosts(self, hosts): if hosts is None: self.hosts = None else: if isinstance(hosts, list): self.hosts = hosts else: self.hosts = [ hosts ] return self def set_prefix(self, prefix): self.prefix = prefix return self def set_forest_count(self, forests): self.forests = int(forests) return self def set_start_number(self, start): self.start = start return self def set_failover(self, failover): self.failover = failover return self def set_failover_host(self, failhost): self.failhost = failhost return self def set_user(self, adminuser): self.adminuser = adminuser return self def set_pass(self, adminpass): self.adminpass = adminpass return self def set_data_dir(self, data_dir): self.data_dir = data_dir return self def set_large_data_dir(self, large_data_dir): self.large_data_dir = large_data_dir return self def set_fast_data_dir(self, fast_data_dir): self.fast_data_dir = fast_data_dir return self def set_database(self, database): self.database = database return self def set_dry_run(self, dry_run): self.dry_run = dry_run return self def create_forests(self): conn = Connection(self.host, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) if self.failhost is None and self.failover != "none": print("Invalid configuration, specify failover-host for failover:", self.failover) sys.exit(1) if self.hosts is None: host_names = self.marklogic.hosts() else: host_names = self.hosts exists = [] host_index = 0 for host_name in host_names: host_index += 1 for count in range(1,self.forests + 1): name = self.prefix + "-" + str(host_index) + "-" + str(count) forest = Forest(name, host_name, conn) if forest.exists(): exists.append(host_name + ":" + name) if len(exists) != 0: print("Abort: forest(s) already exist:") for f in exists: print(" ", f) sys.exit(1) host_index = 0 for host_name in host_names: host_index += 1 for count in range(self.start,self.start + self.forests): name = self.prefix + "-" + str(host_index) + "-" + str(count) forest = Forest(name, host_name, conn) if self.data_dir is not None: forest.set_data_directory(self.data_dir) if self.large_data_dir is not None: forest.set_large_data_directory(self.large_data_dir) if self.fast_data_dir is not None: forest.set_fast_data_directory(self.fast_data_dir) if self.failhost is not None: forest.set_failover(self.failover) forest.set_failover_host_names(self.failhost) if self.database is not None: forest.set_database(self.database) print("Create forest " + name + " on " + host_name) if self.dry_run: print(json.dumps(forest.marshal(), sort_keys=True, indent=2)) else: forest.create() print("Finished")
class App: def __init__(self): self.marklogic = None self.uname = pwd.getpwuid(os.getuid()).pw_name self.adminuser = "******" self.adminpass = "******" self.realm = "public" self.host = None self.boothost = None self.couple = None self.couple_user = None self.couple_pass = None self.name = None def set_credentials(self, user, password): self.adminuser = user self.adminpass = password def set_realm(self, realm): self.realm = realm def set_boot_host(self, host): self.boothost = host def set_host(self, name): self.host = name def set_couple(self, couple): self.couple = couple def set_couple_credentials(self, user, password): self.couple_user = user self.couple_pass = password def set_name(self, name): self.name = name def setup_cluster(self): if self.couple is not None and self.couple_pass is None: self.couple_pass = self.adminpass self.couple_user = self.adminuser if self.boothost is None: self.boothost = self.host[0] self.host.remove(self.boothost) if self.ml_init(self.boothost): self.ml_security(self.boothost) conn = Connection(self.boothost, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) for hostname in self.host: self.ml_init(hostname) self.ml_join(self.boothost, hostname) if self.name is not None: print("{0}: rename cluster...".format(self.boothost)) cluster = self.marklogic.cluster() cluster.set_cluster_name(self.name) cluster.update() if self.couple is not None: for couple in self.couple: print("{0}: couple with {1}...".format(self.boothost, couple)) altconn = Connection(couple, HTTPDigestAuth(self.couple_user, self.couple_pass)) altml = MarkLogic(altconn) altcluster = altml.cluster() cluster = self.marklogic.cluster() cluster.couple(altcluster) print("Finished") def ml_init(self, hostname): print("{0}: initialize host...".format(hostname)) try: host = MarkLogic.instance_init(hostname) except UnauthorizedAPIRequest: # Assume that this happened because the host is already initialized host = Host(hostname) return host.just_initialized() def ml_join(self, boothost, hostname): print("{0}: join cluster with {1}...".format(hostname, boothost)) cluster = self.marklogic.cluster() host = Host(hostname) cluster.add_host(host) def ml_security(self, hostname): print("{0}: initialize security...".format(hostname)) MarkLogic.instance_admin(hostname, self.realm, self.adminuser, self.adminpass)
def setup_cluster(self): if self.couple is not None and self.couple_pass is None: self.couple_pass = self.adminpass self.couple_user = self.adminuser self.load_blacklist() self.find_containers() if not self.container_list: print("There must be at least one unused container running.") sys.exit(1) if self.localimage: ps = os.popen("docker exec " + self.container_list[0] + " ip route") for line in ps.readlines(): match = re.match("^default via (\S+)", line) if match: self.localip = match.group(1) if self.localip is None: print("Cannot find IP address of localhost!?") sys.exit(1) # Find the bootstrap image if self.localimage: pass else: self.bootimage = self.pick_image(self.bootimage) if self.bootimage in self.container_list: self.container_list.remove(self.bootimage) self.cluster_list = self.pick_containers() self.display_info() #sys.exit(1) # Initialize the bootstrap image, if necessary if self.localimage: bootip = self.localip else: bootip = self.ipaddr[self.bootimage] if self.ml_init(bootip, self.bootimage): self.ml_security(bootip, self.bootimage) conn = Connection(bootip, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) for container in self.cluster_list: if container == self.bootimage: continue ip = self.ipaddr[container] self.ml_init(ip, container) self.ml_join(bootip, ip) if self.name is not None: print("{0}: rename cluster...".format(bootip)) cluster = self.marklogic.cluster() cluster.set_cluster_name(self.name) cluster.update() if self.couple is not None: for couple in self.couple: print("{0}: couple with {1}...".format(bootip, couple)) altconn = Connection( couple, HTTPDigestAuth(self.couple_user, self.couple_pass)) altml = MarkLogic(altconn) altcluster = altml.cluster() cluster = self.marklogic.cluster() cluster.couple(altcluster) print("Finished")
def backup_databases(self): conn = Connection(self.host, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) actual_databases = self.marklogic.databases() if self.databases is None: self.databases = actual_databases if self.backup_root is None: raise UnsupportedOperation("You must specify the backup root.") if self.max_parallel <= 0: self.max_parallel = 1 for dbname in self.databases: if not dbname in actual_databases: raise UnsupportedOperation("Database does not exist: {0}" .format(dbname)) maxp = self.max_parallel running = 0 done = False queue = self.databases status_list = {} min_wait = 5 max_wait = 30 wait_incr = 5 wait = min_wait while not done: done = True while len(queue) > 0 and running < maxp: dbname = queue.pop(0) running += 1 done = False print("Backing up {0}".format(dbname)) if not self.dry_run: db = self.marklogic.database(dbname) bkp = db.backup(self.backup_root + dbname, connection=conn) status_list[dbname] = bkp response = bkp.status() if response['status'] != 'in-progress': print("{0}: {1}".format(dbname, response['status'])) if self.dry_run: if running > 0 or len(queue) > 0: print("{0} backups in dry-run; {1} in queue..." .format(running, len(queue))) running = 0 else: if len(status_list) > 0: new_list = {} for dbname in status_list: bkp = status_list[dbname] response = bkp.status() print("{0}: {1}".format(dbname, response['status'])) if response['status'] == 'in-progress': done = False new_list[dbname] = bkp else: running -= 1 wait = min_wait done = done and len(queue) == 0 if not done: status_list = new_list if running < maxp and len(queue) != 0: print("Running: {0} backups running; {1} in queue..." .format(running, len(queue))) wait = min_wait print("") else: print("Waiting {0}s: {1} backups running; {2} in queue..." .format(wait, running, len(queue))) time.sleep(wait) if wait < max_wait: wait += wait_incr print("")
class InitServer: def __init__(self): pass #logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser() parser.add_argument("--host", action='store', default="localhost", help="Management API host") parser.add_argument("--username", action='store', default="admin", help="User name") parser.add_argument("--password", action='store', default="admin", help="Password") parser.add_argument("--wallet", action='store', default="admin", help="Wallet password") parser.add_argument('--debug', action='store_true', help='Enable debug logging') args = parser.parse_args() if args.debug: logging.basicConfig(level=logging.WARNING) logging.getLogger("requests").setLevel(logging.WARNING) logging.getLogger("marklogic").setLevel(logging.DEBUG) print("Initialize host {}".format(args.host)) MarkLogic.instance_init(args.host) print("Initialize admin {}".format(args.host)) MarkLogic.instance_admin(args.host, "public", args.username, args.password, args.wallet) print("finished")
def __init__(self, argv): super().__init__(argv) self.connection = Connection(tc.hostname, HTTPDigestAuth(tc.admin, tc.password)) self.marklogic = MarkLogic(self.connection)
class Docker: def __init__(self): self.marklogic = None self.uname=pwd.getpwuid(os.getuid()).pw_name self.adminuser="******" self.adminpass="******" self.realm="public" self.imatch="ml[0-9]" self.bootimage=None self.localimage=False self.count=None self.couple=None self.couple_user=None self.couple_pass=None self.name=None self.blacklist_file="/tmp/{0}.docker.skip".format(self.uname) self.blacklist={} self.container_list=[] self.cluster_list=[] self.containers={} self.localip=None self.ipaddr={} self.hostname={} def set_credentials(self,user,password): self.adminuser = user self.adminpass = password def set_realm(self,realm): self.realm = realm def set_image_match(self,match): self.imatch = match def set_boot_image(self,image): self.bootimage = image self.localimage = (image == "localhost") def set_count(self,count): self.count = count def set_couple(self,couple): self.couple = couple def set_couple_credentials(self,user,password): self.couple_user = user self.couple_pass = password def set_name(self,name): self.name = name def setup_cluster(self): if self.couple is not None and self.couple_pass is None: self.couple_pass = self.adminpass self.couple_user = self.adminuser self.load_blacklist() self.find_containers() if not self.container_list: print("There must be at least one unused container running.") sys.exit(1) if self.localimage: ps = os.popen("docker exec " + self.container_list[0] + " ip route") for line in ps.readlines(): match = re.match("^default via (\S+)", line) if match: self.localip = match.group(1) if self.localip is None: print("Cannot find IP address of localhost!?") sys.exit(1) # Find the bootstrap image if self.localimage: pass else: self.bootimage = self.pick_image(self.bootimage) self.container_list.remove(self.bootimage) self.cluster_list = self.pick_containers() self.display_info() #sys.exit(1) # Initialize the bootstrap image, if necessary if self.localimage: bootip = self.localip else: bootip = self.ipaddr[self.bootimage] if self.ml_init(bootip, self.bootimage): self.ml_security(bootip, self.bootimage) conn = Connection(bootip, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) for container in self.cluster_list: if container == self.bootimage: continue ip = self.ipaddr[container] self.ml_init(ip, container) self.ml_join(bootip, ip) if self.name is not None: print("{0}: rename cluster...".format(bootip)) cluster = self.marklogic.cluster() cluster.set_cluster_name(self.name) cluster.update() if self.couple is not None: for couple in self.couple: print("{0}: couple with {1}...".format(bootip,couple)) altconn = Connection(couple, HTTPDigestAuth(self.couple_user, self.couple_pass)) altml = MarkLogic(altconn) altcluster = altml.cluster() cluster = self.marklogic.cluster() cluster.couple(altcluster) print("Finished") def ml_init(self, ip, container): if container in self.hostname: hostname = self.hostname[container] else: hostname = container print("{0}: initialize host {1}...".format(ip,hostname)) try: host = MarkLogic.instance_init(ip) except UnauthorizedAPIRequest: # Assume that this happened because the host is already initialized host = Host(ip) self.blacklist[container] = "used" self.save_blacklist() return host.just_initialized() def ml_join(self, bootip, ip): print("{0}: join cluster with {1}...".format(ip,bootip)) cluster = self.marklogic.cluster() host = Host(ip) cluster.add_host(host) def ml_security(self, ip, container): print("{0}: initialize security...".format(ip)) MarkLogic.instance_admin(ip,self.realm,self.adminuser,self.adminpass) self.blacklist[container] = "boot" self.save_blacklist() def pick_image(self, name): if name is None: for container in self.container_list: if not container in self.blacklist: return container print("Cannot find unused container") sys.exit(1) for key in self.hostname: if name == self.hostname[key]: return key count = 0 container = None for key in self.ipaddr: if name == self.ipaddr[key]: return key if re.match("^"+name, key): count += 1 container = key if count == 1: return container if count > 1: print("Ambiguous container id:", name) sys.exit(1) for key in self.containers: image = self.containers[key] if (key == self.bootimage or re.match(self.bootimage, image)): return key print("Cannot find container \"{0}\".".format(name)) sys.exit(1) def pick_containers(self): if self.count is None: count = -1 else: count = self.count - 1 cset = set() if not self.localimage: cset.add(self.bootimage) for container in self.container_list: if container == self.bootimage or container in self.blacklist: continue if count == 0: continue cset.add(container) count -= 1 containers = [] for key in cset: containers.append(key) return containers def load_blacklist(self): try: f = open(self.blacklist_file, "r") for line in f.readlines(): (container, flag) = line.strip().split(" ") self.blacklist[container] = flag f.close() except FileNotFoundError: pass def save_blacklist(self): f = open(self.blacklist_file, "w") for key in self.blacklist: if self.blacklist[key] != "gone": f.write("{0} {1}\n".format(key,self.blacklist[key])) f.close() def find_containers(self): self.load_blacklist() ps = os.popen("docker ps") for line in ps.readlines(): match = re.match("([0-9a-f]+)\s+(\S+)", line) if match: container = match.group(1) image = match.group(2) ipx = os.popen("docker inspect {0}".format(container)) ip = None hostname = None for line in ipx.readlines(): match = re.match('\s+"IPAddress": "([^"]+)"', line) if match: ip = match.group(1) match = re.match('\s+"Hostname": "([^"]+)"', line) if match: hostname = match.group(1) if ip is None: print("Cannot read IP address of container {0}" .format(container)) else: self.ipaddr[container] = ip if hostname is None: print("Cannot read hostname of container {0}" .format(container)) else: self.hostname[container] = hostname if re.match(self.imatch, image): self.container_list = [container] + self.container_list self.containers[container] = image for container in self.blacklist: if not container in self.containers: self.blacklist[container] = "gone" def display_info(self): # Display information about docker containers # Sigh. Report formatting is such a drag... dinfo = [] ps = os.popen("docker ps") for line in ps.readlines(): match = re.match("([0-9a-f]+)\s+(\S+)", line) if match: container = match.group(1) image = match.group(2) ip = self.ipaddr[container] hostname = self.hostname[container] attr = [] if container in self.blacklist: attr.append("used") if self.blacklist[container] == "boot": attr.append("boot") else: if not container in self.cluster_list: attr.append("skip") else: if re.match(self.imatch, image): attr.append("*") if container == self.bootimage: attr.append("boot") else: attr.append("!~"+self.imatch) data = {"container": container, "image": image, "ipaddr": ip, "hostname": hostname, "flags": ", ".join(attr)} dinfo = [data] + dinfo headers = {"container": "Container", "image": "Image", "hostname": "Hostname", "ipaddr": "IP Addr", "flags": "Flags"} maxlen = {} for key in dinfo[0]: flen = len(headers[key]) for data in dinfo: if len(data[key]) > flen: flen = len(data[key]) maxlen[key] = flen fstr = "%-{0}s %-{1}s %-{2}s %-{3}s %s".format( maxlen['container'], maxlen['image'], maxlen['hostname'], maxlen['ipaddr']) print(fstr % (headers['container'], headers['image'], headers['hostname'], headers['ipaddr'], headers['flags'])) for data in dinfo: print(fstr % (data['container'], data['image'], data['hostname'], data['ipaddr'], data['flags']))
def ml_security(self, ip, container): print("{0}: initialize security...".format(ip)) MarkLogic.instance_admin(ip,self.realm,self.adminuser,self.adminpass) self.blacklist[container] = "boot" self.save_blacklist()
def ml_security(self, ip, container): print("{0}: initialize security...".format(ip)) MarkLogic.instance_admin(ip, self.realm, self.adminuser, self.adminpass) self.blacklist[container] = "boot" self.save_blacklist()
def restore_databases(self): conn = Connection(self.host, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) actual_databases = self.marklogic.databases() if self.databases is None: self.databases = actual_databases if self.restore_root is None: raise UnsupportedOperation("You must specify the restore root.") if self.max_parallel <= 0: self.max_parallel = 1 for dbname in self.databases: if not dbname in actual_databases: raise UnsupportedOperation("Database does not exist: {0}" .format(dbname)) maxp = self.max_parallel running = 0 done = False queue = self.databases status_list = {} min_wait = 5 max_wait = 30 wait_incr = 5 wait = min_wait while not done: done = True while len(queue) > 0 and running < maxp: dbname = queue.pop(0) running += 1 done = False print("Restoring {0}".format(dbname)) if not self.dry_run: db = self.marklogic.database(dbname) rst = db.restore(self.restore_root + dbname, connection=conn) status_list[dbname] = rst response = rst.status() for forest in response['forest']: if forest['status'] != 'in-progress': print("Forest {0}: {1}" .format(forest['forest-name'], forest['status'])) if self.dry_run: if running > 0 or len(queue) > 0: print("{0} restores in dry-run; {1} in queue..." .format(running, len(queue))) running = 0 else: if len(status_list) > 0: new_list = {} for dbname in status_list: rst = status_list[dbname] response = rst.status() fdone = True for forest in response['forest']: print("Forest {0}: {1}" .format(forest['forest-name'], forest['status'])) if forest['status'] == 'in-progress': fdone = False if not fdone: done = False new_list[dbname] = rst else: running -= 1 wait = min_wait done = done and len(queue) == 0 if not done: status_list = new_list if running < maxp and len(queue) != 0: print("Running: {0} restores running; {1} in queue..." .format(running, len(queue))) wait = min_wait print("") else: print("Waiting {0}s: {1} restores running; {2} in queue..." .format(wait, running, len(queue))) time.sleep(wait) if wait < max_wait: wait += wait_incr print("")
class RestoreDatabases: def __init__(self): self.marklogic = None self.host = "localhost" self.adminuser = "******" self.adminpass = "******" self.databases = None self.restore_root = None self.journal_arch = False self.lag_limit = 30 self.incremental = False self.max_parallel = 5 self.dry_run = False # TODO: better checking of argument types def set_host(self, host): self.host = host return self def set_user(self, adminuser): self.adminuser = adminuser return self def set_pass(self, adminpass): self.adminpass = adminpass return self def set_databases(self, databases): self.databases = databases def set_restore_root(self, directory): if directory is not None: self.restore_root = directory if not self.restore_root.endswith("/"): self.restore_root += "/" return self def set_journal_archiving(self, archiving): self.journal_arch = archiving return self def set_lag_limit(self, limit): self.lag_limit = limit return self def set_incremental(self, incremental): self.incremental = incremental return self def set_max_parallel(self, max_parallel): self.max_parallel = max_parallel return self def set_dry_run(self, dry_run): self.dry_run = dry_run return self def restore_databases(self): conn = Connection(self.host, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) actual_databases = self.marklogic.databases() if self.databases is None: self.databases = actual_databases if self.restore_root is None: raise UnsupportedOperation("You must specify the restore root.") if self.max_parallel <= 0: self.max_parallel = 1 for dbname in self.databases: if not dbname in actual_databases: raise UnsupportedOperation("Database does not exist: {0}" .format(dbname)) maxp = self.max_parallel running = 0 done = False queue = self.databases status_list = {} min_wait = 5 max_wait = 30 wait_incr = 5 wait = min_wait while not done: done = True while len(queue) > 0 and running < maxp: dbname = queue.pop(0) running += 1 done = False print("Restoring {0}".format(dbname)) if not self.dry_run: db = self.marklogic.database(dbname) rst = db.restore(self.restore_root + dbname, connection=conn) status_list[dbname] = rst response = rst.status() for forest in response['forest']: if forest['status'] != 'in-progress': print("Forest {0}: {1}" .format(forest['forest-name'], forest['status'])) if self.dry_run: if running > 0 or len(queue) > 0: print("{0} restores in dry-run; {1} in queue..." .format(running, len(queue))) running = 0 else: if len(status_list) > 0: new_list = {} for dbname in status_list: rst = status_list[dbname] response = rst.status() fdone = True for forest in response['forest']: print("Forest {0}: {1}" .format(forest['forest-name'], forest['status'])) if forest['status'] == 'in-progress': fdone = False if not fdone: done = False new_list[dbname] = rst else: running -= 1 wait = min_wait done = done and len(queue) == 0 if not done: status_list = new_list if running < maxp and len(queue) != 0: print("Running: {0} restores running; {1} in queue..." .format(running, len(queue))) wait = min_wait print("") else: print("Waiting {0}s: {1} restores running; {2} in queue..." .format(wait, running, len(queue))) time.sleep(wait) if wait < max_wait: wait += wait_incr print("")
class BackupDatabases: def __init__(self): self.marklogic = None self.host = "localhost" self.adminuser = "******" self.adminpass = "******" self.databases = None self.backup_root = None self.journal_arch = False self.lag_limit = 30 self.incremental = False self.max_parallel = 5 self.dry_run = False # TODO: better checking of argument types def set_host(self, host): self.host = host return self def set_user(self, adminuser): self.adminuser = adminuser return self def set_pass(self, adminpass): self.adminpass = adminpass return self def set_databases(self, databases): self.databases = databases def set_backup_root(self, directory): if directory is not None: self.backup_root = directory if not self.backup_root.endswith("/"): self.backup_root += "/" return self def set_journal_archiving(self, archiving): self.journal_arch = archiving return self def set_lag_limit(self, limit): self.lag_limit = limit return self def set_incremental(self, incremental): self.incremental = incremental return self def set_max_parallel(self, max_parallel): self.max_parallel = max_parallel return self def set_dry_run(self, dry_run): self.dry_run = dry_run return self def backup_databases(self): conn = Connection(self.host, HTTPDigestAuth(self.adminuser, self.adminpass)) self.marklogic = MarkLogic(conn) actual_databases = self.marklogic.databases() if self.databases is None: self.databases = actual_databases if self.backup_root is None: raise UnsupportedOperation("You must specify the backup root.") if self.max_parallel <= 0: self.max_parallel = 1 for dbname in self.databases: if not dbname in actual_databases: raise UnsupportedOperation("Database does not exist: {0}" .format(dbname)) maxp = self.max_parallel running = 0 done = False queue = self.databases status_list = {} min_wait = 5 max_wait = 30 wait_incr = 5 wait = min_wait while not done: done = True while len(queue) > 0 and running < maxp: dbname = queue.pop(0) running += 1 done = False print("Backing up {0}".format(dbname)) if not self.dry_run: db = self.marklogic.database(dbname) bkp = db.backup(self.backup_root + dbname, connection=conn) status_list[dbname] = bkp response = bkp.status() if response['status'] != 'in-progress': print("{0}: {1}".format(dbname, response['status'])) if self.dry_run: if running > 0 or len(queue) > 0: print("{0} backups in dry-run; {1} in queue..." .format(running, len(queue))) running = 0 else: if len(status_list) > 0: new_list = {} for dbname in status_list: bkp = status_list[dbname] response = bkp.status() print("{0}: {1}".format(dbname, response['status'])) if response['status'] == 'in-progress': done = False new_list[dbname] = bkp else: running -= 1 wait = min_wait done = done and len(queue) == 0 if not done: status_list = new_list if running < maxp and len(queue) != 0: print("Running: {0} backups running; {1} in queue..." .format(running, len(queue))) wait = min_wait print("") else: print("Waiting {0}s: {1} backups running; {2} in queue..." .format(wait, running, len(queue))) time.sleep(wait) if wait < max_wait: wait += wait_incr print("")