def max_timeLimit(self, connection=None):
if connection is None:
connection = self.connection
# Work out which appserver we're running against
# 1. It's usually 'App-Services'
marklogic = MarkLogic(connection)
server = marklogic.http_server('App-Services')
if server.exists() and server.port() == connection.port:
pass
else:
slist = marklogic.http_servers()
server = None
while slist and server is None:
group, name = re.split("\\|", slist.pop())
try:
check = marklogic.http_server(name, group=group)
if check.port() == connection.port:
server = check
except UnexpectedManagementAPIResponse:
pass
except:
raise
if (server is not None and server.exists() \
and server.port() == connection.port):
return server.max_time_limit()
else:
# Oh, heck, just go with a default
return 600
sys.exit(1)
def stop(self, args, config, connection):
status = self.status(args, config, connection, internal=True)
if status == 'down':
print("MarkLogic server on {0} appears to be {1}.".format(
connection.host, status))
else:
if 'cluster' in args:
hostname = connection.host
if hostname == 'localhost':
hostname = socket.gethostname()
cluster = LocalCluster(connection=connection).read()
print("Shutting down cluster...")
cluster.shutdown()
else:
hostname = connection.host
if hostname == 'localhost':
hostname = socket.gethostname()
host = Host(hostname, connection=connection).read()
if host.group_name() is None:
ml = MarkLogic(connection)
if len(ml.hosts()) == 1:
host = Host(ml.hosts()[0],
connection=connection).read()
else:
print("Failed to identify host:", ml.hosts())
sys.exit(1)
print("Shutting down host: " + host.host_name())
host.shutdown()
status = self.status(args, config, connection, internal=True)
while status == 'up':
time.sleep(2)
status = self.status(args, config, connection, internal=True)
def stop(self, args, config, connection):
status = self.status(args, config, connection, internal=True)
if status == 'down':
print("MarkLogic server on {0} appears to be {1}."
.format(connection.host, status))
else:
if 'cluster' in args:
hostname = connection.host
if hostname == 'localhost':
hostname = socket.gethostname()
cluster = LocalCluster(connection=connection).read()
print("Shutting down cluster...")
cluster.shutdown()
else:
hostname = connection.host
if hostname == 'localhost':
hostname = socket.gethostname()
host = Host(hostname,connection=connection).read()
if host.group_name() is None:
ml = MarkLogic(connection)
if len(ml.hosts()) == 1:
host = Host(ml.hosts()[0], connection=connection).read()
else:
print("Failed to identify host:",ml.hosts())
sys.exit(1)
print("Shutting down host: " + host.host_name())
host.shutdown()
status = self.status(args,config,connection,internal=True)
while status == 'up':
time.sleep(2)
status = self.status(args,config,connection,internal=True)
def join(self, args, config, connection):
cluster = LocalCluster(connection=connection)
cluster.read()
self.logger.info("Initializing {0}...".format(args['host']))
MarkLogic.instance_init(args['host'])
host = Host(args['host'])
self.logger.info("Joining cluster...")
cluster.add_host(host)
def join(self, args, config, connection):
cluster = LocalCluster(connection=connection)
cluster.read()
self.logger.info("Initializing {0}...".format(args['host']))
MarkLogic.instance_init(args['host'])
host = Host(args['host'])
self.logger.info("Joining cluster...")
cluster.add_host(host)
def test_create_database(self):
self.marklogic = MarkLogic(self.connection)
dbname = 'test-mma-database'
mma = MMA(self.connection)
mma.run(['create', 'database', dbname])
assert dbname in self.marklogic.databases()
mma.run(['delete', 'database', dbname])
assert dbname not in self.marklogic.databases()
class TestMmaDatabase(MLConfig):
def test_create_database(self):
self.marklogic = MarkLogic(self.connection)
dbname = 'test-mma-database'
mma = MMA(self.connection)
mma.run(['create', 'database', dbname])
assert dbname in self.marklogic.databases()
mma.run(['delete', 'database', dbname])
assert dbname not in self.marklogic.databases()
def couple_cluster(self):
conn = Connection(self.host, HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
if self.couple is not None:
for couple in self.couple:
print("{0}: couple with {1}...".format(self.host,couple))
altconn = Connection(couple,
HTTPDigestAuth(self.couple_user,
self.couple_pass))
altml = MarkLogic(altconn)
altcluster = altml.cluster()
cluster = self.marklogic.cluster()
cluster.couple(altcluster)
print("Finished")
def couple_cluster(self):
conn = Connection(self.host,
HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
if self.couple is not None:
for couple in self.couple:
print("{0}: couple with {1}...".format(self.host, couple))
altconn = Connection(
couple, HTTPDigestAuth(self.couple_user, self.couple_pass))
altml = MarkLogic(altconn)
altcluster = altml.cluster()
cluster = self.marklogic.cluster()
cluster.couple(altcluster)
print("Finished")
def build_cluster(host, adminuser, adminpass, couple_user, couple_pass, cluster_host_list):
conn = Connection(host, HTTPDigestAuth(adminuser, adminpass))
marklogic = MarkLogic(conn)
if cluster_host_list is not None:
for couple in cluster_host_list:
print("{0}: couple with {1}...".format(host, couple))
altconn = Connection(couple,
HTTPDigestAuth(couple_user,
couple_pass))
altml = MarkLogic(altconn)
altcluster = altml.cluster()
cluster = marklogic.cluster()
cluster.couple(altcluster)
print("Finished")
def init(self, args, config, connection):
status = self.status(args,config,connection,internal=True)
if status != 'up':
try:
data = config[args['config']]['datadir']
print("Clearing {0}...".format(data))
shutil.rmtree(data)
os.mkdir(data)
except KeyError:
pass
self.start(args,config,connection)
print("Initializing {0}...".format(connection.host))
MarkLogic.instance_init(connection.host)
MarkLogic.instance_admin(connection.host, args['realm'],
connection.auth.username,
connection.auth.password)
def init(self, args, config, connection):
status = self.status(args, config, connection, internal=True)
if status != 'up':
try:
data = config[args['config']]['datadir']
print("Clearing {0}...".format(data))
shutil.rmtree(data)
os.mkdir(data)
except KeyError:
pass
self.start(args, config, connection)
print("Initializing {0}...".format(connection.host))
MarkLogic.instance_init(connection.host)
MarkLogic.instance_admin(connection.host, args['realm'],
connection.auth.username,
connection.auth.password)
def create_forests(self):
conn = Connection(self.host,
HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
if self.failhost is None and self.failover != "none":
print("Invalid configuration, specify failover-host for failover:",
self.failover)
sys.exit(1)
if self.hosts is None:
host_names = self.marklogic.hosts()
else:
host_names = self.hosts
exists = []
host_index = 0
for host_name in host_names:
host_index += 1
for count in range(1,self.forests + 1):
name = self.prefix + "-" + str(host_index) + "-" + str(count)
forest = Forest(name, host_name, conn)
if forest.exists():
exists.append(host_name + ":" + name)
if len(exists) != 0:
print("Abort: forest(s) already exist:")
for f in exists:
print(" ", f)
sys.exit(1)
host_index = 0
for host_name in host_names:
host_index += 1
for count in range(self.start,self.start + self.forests):
name = self.prefix + "-" + str(host_index) + "-" + str(count)
forest = Forest(name, host_name, conn)
if self.data_dir is not None:
forest.set_data_directory(self.data_dir)
if self.large_data_dir is not None:
forest.set_large_data_directory(self.large_data_dir)
if self.fast_data_dir is not None:
forest.set_fast_data_directory(self.fast_data_dir)
if self.failhost is not None:
forest.set_failover(self.failover)
forest.set_failover_host_names(self.failhost)
if self.database is not None:
forest.set_database(self.database)
print("Create forest " + name + " on " + host_name)
if self.dry_run:
print(json.dumps(forest.marshal(), sort_keys=True, indent=2))
else:
forest.create()
print("Finished")
def ml_init(self, hostname):
print("{0}: initialize host...".format(hostname))
try:
host = MarkLogic.instance_init(hostname)
except UnauthorizedAPIRequest:
# Assume that this happened because the host is already initialized
host = Host(hostname)
return host.just_initialized()
def ml_init(self, hostname):
print("{0}: initialize host...".format(hostname))
try:
host = MarkLogic.instance_init(hostname)
except UnauthorizedAPIRequest:
# Assume that this happened because the host is already initialized
host = Host(hostname)
return host.just_initialized()
class TestMmaDatabase(unittest.TestCase):
"""
Basic creation test function.
"""
def __init__(self, argv):
super().__init__(argv)
self.connection = Connection(tc.hostname,
HTTPDigestAuth(tc.admin, tc.password))
self.marklogic = MarkLogic(self.connection)
def test_create_database(self):
dbname = 'test-mma-database'
mma = MMA(self.connection)
mma.run(['create','database',dbname])
self.assertTrue(dbname in self.marklogic.databases())
mma.run(['delete','database',dbname])
self.assertFalse(dbname in self.marklogic.databases())
def debug(self, args, config, connection):
section = config[args['config']]
if 'diagnostic-events' in section:
events = []
devents = section['diagnostic-events'].strip()
if devents != '':
for event in devents.split(","):
events.append(event.strip())
else:
events = []
ml = MarkLogic(connection)
group = ml.group(args['group'])
group.set_events(events)
if len(events) == 0:
group.set_events_activated(False)
else:
group.set_events_activated(True)
group.update()
def init(self, args, config, connection):
status = self.status(args,config,connection,internal=True)
if status == 'up':
print("Cannot initialize a running host!")
sys.exit(1)
else:
if connection.host == 'localhost':
try:
data = config[args['config']]['datadir']
print("Clearing {0}...".format(data))
self._clear_directory(data)
except KeyError:
pass
else:
self.logger.info("Skipping clear data directory; not localhost")
self.start(args,config,connection)
print("Initializing {0}...".format(connection.host))
MarkLogic.instance_init(connection.host)
MarkLogic.instance_admin(connection.host, args['realm'],
connection.auth.username,
connection.auth.password)
def debug(self, args, config, connection):
section = config[args['config']]
if 'diagnostic-events' in section:
events = []
devents = section['diagnostic-events'].strip()
if devents != '':
for event in devents.split(","):
events.append(event.strip())
else:
events = []
ml = MarkLogic(connection)
group = ml.group(args['group'])
group.set_events(events)
if len(events) == 0:
group.set_events_activated(False)
else:
group.set_events_activated(True)
group.update()
def debug(self, args, config, connection):
section = config[args['config']]
if 'diagnostic-events' in section:
events = section['diagnostic-events'].strip()
if events == '':
events = []
else:
events = re.sub(r'\s+', '', events)
events = events.split(',')
else:
events = []
ml = MarkLogic(connection)
group = ml.group(args['group'])
group.set_events(events)
if len(events) == 0:
group.set_events_activated(False)
else:
group.set_events_activated(True)
group.update()
def debug(self, args, config, connection):
section = config[args['config']]
if 'diagnostic-events' in section:
events = section['diagnostic-events'].strip()
if events == '':
events = []
else:
events = re.sub(r'\s+', '', events)
events = events.split(',')
else:
events = []
ml = MarkLogic(connection)
group = ml.group(args['group'])
group.set_events(events)
if len(events) == 0:
group.set_events_activated(False)
else:
group.set_events_activated(True)
group.update()
def init(self, args, config, connection):
status = self.status(args, config, connection, internal=True)
if status == 'up':
print("Cannot initialize a running host!")
sys.exit(1)
else:
if connection.host == 'localhost':
try:
data = config[args['config']]['datadir']
print("Clearing {0}...".format(data))
self._clear_directory(data)
except KeyError:
pass
else:
self.logger.info(
"Skipping clear data directory; not localhost")
self.start(args, config, connection)
print("Initializing {0}...".format(connection.host))
MarkLogic.instance_init(connection.host)
MarkLogic.instance_admin(connection.host, args['realm'],
connection.auth.username,
connection.auth.password)
def setup_cluster(self):
if self.couple is not None and self.couple_pass is None:
self.couple_pass = self.adminpass
self.couple_user = self.adminuser
if self.boothost is None:
self.boothost = self.host[0]
self.host.remove(self.boothost)
if self.ml_init(self.boothost):
self.ml_security(self.boothost)
conn = Connection(self.boothost, HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
for hostname in self.host:
self.ml_init(hostname)
self.ml_join(self.boothost, hostname)
if self.name is not None:
print("{0}: rename cluster...".format(self.boothost))
cluster = self.marklogic.cluster()
cluster.set_cluster_name(self.name)
cluster.update()
if self.couple is not None:
for couple in self.couple:
print("{0}: couple with {1}...".format(self.boothost, couple))
altconn = Connection(couple,
HTTPDigestAuth(self.couple_user,
self.couple_pass))
altml = MarkLogic(altconn)
altcluster = altml.cluster()
cluster = self.marklogic.cluster()
cluster.couple(altcluster)
print("Finished")
def setup_cluster(self):
if self.couple is not None and self.couple_pass is None:
self.couple_pass = self.adminpass
self.couple_user = self.adminuser
if self.boothost is None:
self.boothost = self.host[0]
self.host.remove(self.boothost)
if self.ml_init(self.boothost):
self.ml_security(self.boothost)
conn = Connection(self.boothost,
HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
for hostname in self.host:
self.ml_init(hostname)
self.ml_join(self.boothost, hostname)
if self.name is not None:
print("{0}: rename cluster...".format(self.boothost))
cluster = self.marklogic.cluster()
cluster.set_cluster_name(self.name)
cluster.update()
if self.couple is not None:
for couple in self.couple:
print("{0}: couple with {1}...".format(self.boothost, couple))
altconn = Connection(
couple, HTTPDigestAuth(self.couple_user, self.couple_pass))
altml = MarkLogic(altconn)
altcluster = altml.cluster()
cluster = self.marklogic.cluster()
cluster.couple(altcluster)
print("Finished")
def ml_init(self, ip, container):
if container in self.hostname:
hostname = self.hostname[container]
else:
hostname = container
print("{0}: initialize host {1}...".format(ip,hostname))
try:
host = MarkLogic.instance_init(ip)
except UnauthorizedAPIRequest:
# Assume that this happened because the host is already initialized
host = Host(ip)
self.blacklist[container] = "used"
self.save_blacklist()
return host.just_initialized()
def ml_init(self, ip, container):
if container in self.hostname:
hostname = self.hostname[container]
else:
hostname = container
print("{0}: initialize host {1}...".format(ip, hostname))
try:
host = MarkLogic.instance_init(ip)
except UnauthorizedAPIRequest:
# Assume that this happened because the host is already initialized
host = Host(ip)
self.blacklist[container] = "used"
self.save_blacklist()
return host.just_initialized()
def connect(self, args):
try:
adminuser, adminpass = re.split(":", args['credentials'])
except ValueError:
print("--credentials value must be 'user:password':"******"requests").setLevel(logging.WARNING)
logging.getLogger("marklogic").setLevel(logging.DEBUG)
self.connection \
= Connection(args['hostname'], HTTPDigestAuth(adminuser, adminpass))
self.mls = MarkLogic(self.connection)
self.args = args
class Couple:
def __init__(self):
self.marklogic = None
self.adminuser = "******"
self.adminpass = "******"
self.host = "localhost"
self.couple = None
self.couple_user = None
self.couple_pass = None
def set_credentials(self, user, password):
self.adminuser = user
self.adminpass = password
def set_host(self, host):
self.host = host
def set_couple(self, couple):
self.couple = couple
def set_couple_credentials(self, user, password):
self.couple_user = user
self.couple_pass = password
def couple_cluster(self):
conn = Connection(self.host,
HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
if self.couple is not None:
for couple in self.couple:
print("{0}: couple with {1}...".format(self.host, couple))
altconn = Connection(
couple, HTTPDigestAuth(self.couple_user, self.couple_pass))
altml = MarkLogic(altconn)
altcluster = altml.cluster()
cluster = self.marklogic.cluster()
cluster.couple(altcluster)
print("Finished")
class Couple:
def __init__(self):
self.marklogic = None
self.adminuser="******"
self.adminpass="******"
self.host="localhost"
self.couple=None
self.couple_user=None
self.couple_pass=None
def set_credentials(self,user,password):
self.adminuser = user
self.adminpass = password
def set_host(self,host):
self.host = host
def set_couple(self,couple):
self.couple = couple
def set_couple_credentials(self,user,password):
self.couple_user = user
self.couple_pass = password
def couple_cluster(self):
conn = Connection(self.host, HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
if self.couple is not None:
for couple in self.couple:
print("{0}: couple with {1}...".format(self.host,couple))
altconn = Connection(couple,
HTTPDigestAuth(self.couple_user,
self.couple_pass))
altml = MarkLogic(altconn)
altcluster = altml.cluster()
cluster = self.marklogic.cluster()
cluster.couple(altcluster)
print("Finished")
class CreateForests:
def __init__(self, marklogic=None, host="localhost", hosts=None, prefix="F", forests=2, start_forest=1,
failover="none", failover_host=None, adminuser="******", adminpass="******", data_dir=None,
large_data_dir=None, fast_data_dir=None, database=None, dry_run=False):
self.marklogic = marklogic
self.host = host
self.hosts = hosts if isinstance(hosts, list) or None else [hosts]
self.prefix = prefix
self.forests = forests
self.start = start_forest
self.failover = failover
self.failhost = failover_host
self.adminuser = adminuser
self.adminpass = adminpass
self.data_dir = data_dir
self.large_data_dir = large_data_dir
self.fast_data_dir = fast_data_dir
self.database = database
self.dry_run = dry_run
def create_forests(self):
conn = Connection(self.host,
HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
if self.failhost is None and self.failover != "none":
print("Invalid configuration, specify failover-host for failover:",
self.failover)
sys.exit(1)
if self.hosts is None:
host_names = self.marklogic.hosts()
else:
host_names = self.hosts
exists = []
for host_index, host_name in enumerate(host_names):
for count in range(1, self.forests + 1):
name = self.prefix + "-" + str(host_index) + "-" + str(count)
forest = Forest(name, host_name, conn)
if forest.exists():
exists.append(host_name + ":" + name)
if len(exists) != 0:
print("Abort: forest(s) already exist:")
for f in exists:
print(" ", f)
sys.exit(1)
for host_index, host_name in enumerate(host_names):
for count in range(self.start, self.start + self.forests):
name = self.prefix + "-" + str(host_index) + "-" + str(count)
forest = Forest(name, host_name, conn)
if self.data_dir is not None:
forest.set_data_directory(self.data_dir)
if self.large_data_dir is not None:
forest.set_large_data_directory(self.large_data_dir)
if self.fast_data_dir is not None:
forest.set_fast_data_directory(self.fast_data_dir)
if self.failhost is not None:
forest.set_failover(self.failover)
forest.set_failover_host_names(self.failhost)
if self.database is not None:
forest.set_database(self.database)
print("Create forest " + name + " on " + host_name)
if self.dry_run:
print(json.dumps(forest.marshal(), sort_keys=True, indent=2))
else:
forest.create()
print("Finished")
def setup_cluster(self):
if self.couple is not None and self.couple_pass is None:
self.couple_pass = self.adminpass
self.couple_user = self.adminuser
self.load_blacklist()
self.find_containers()
if not self.container_list:
print("There must be at least one unused container running.")
sys.exit(1)
if self.localimage:
ps = os.popen("docker exec " + self.container_list[0] + " ip route")
for line in ps.readlines():
match = re.match("^default via (\S+)", line)
if match:
self.localip = match.group(1)
if self.localip is None:
print("Cannot find IP address of localhost!?")
sys.exit(1)
# Find the bootstrap image
if self.localimage:
pass
else:
self.bootimage = self.pick_image(self.bootimage)
self.container_list.remove(self.bootimage)
self.cluster_list = self.pick_containers()
self.display_info()
#sys.exit(1)
# Initialize the bootstrap image, if necessary
if self.localimage:
bootip = self.localip
else:
bootip = self.ipaddr[self.bootimage]
if self.ml_init(bootip, self.bootimage):
self.ml_security(bootip, self.bootimage)
conn = Connection(bootip, HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
for container in self.cluster_list:
if container == self.bootimage:
continue
ip = self.ipaddr[container]
self.ml_init(ip, container)
self.ml_join(bootip, ip)
if self.name is not None:
print("{0}: rename cluster...".format(bootip))
cluster = self.marklogic.cluster()
cluster.set_cluster_name(self.name)
cluster.update()
if self.couple is not None:
for couple in self.couple:
print("{0}: couple with {1}...".format(bootip,couple))
altconn = Connection(couple,
HTTPDigestAuth(self.couple_user,
self.couple_pass))
altml = MarkLogic(altconn)
altcluster = altml.cluster()
cluster = self.marklogic.cluster()
cluster.couple(altcluster)
print("Finished")
class App:
def __init__(self):
self.marklogic = None
self.uname = pwd.getpwuid(os.getuid()).pw_name
self.adminuser = "******"
self.adminpass = "******"
self.realm = "public"
self.host = None
self.boothost = None
self.couple = None
self.couple_user = None
self.couple_pass = None
self.name = None
def set_credentials(self, user, password):
self.adminuser = user
self.adminpass = password
def set_realm(self, realm):
self.realm = realm
def set_boot_host(self, host):
self.boothost = host
def set_host(self, name):
self.host = name
def set_couple(self, couple):
self.couple = couple
def set_couple_credentials(self, user, password):
self.couple_user = user
self.couple_pass = password
def set_name(self, name):
self.name = name
def setup_cluster(self):
if self.couple is not None and self.couple_pass is None:
self.couple_pass = self.adminpass
self.couple_user = self.adminuser
if self.boothost is None:
self.boothost = self.host[0]
self.host.remove(self.boothost)
if self.ml_init(self.boothost):
self.ml_security(self.boothost)
conn = Connection(self.boothost,
HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
for hostname in self.host:
self.ml_init(hostname)
self.ml_join(self.boothost, hostname)
if self.name is not None:
print("{0}: rename cluster...".format(self.boothost))
cluster = self.marklogic.cluster()
cluster.set_cluster_name(self.name)
cluster.update()
if self.couple is not None:
for couple in self.couple:
print("{0}: couple with {1}...".format(self.boothost, couple))
altconn = Connection(
couple, HTTPDigestAuth(self.couple_user, self.couple_pass))
altml = MarkLogic(altconn)
altcluster = altml.cluster()
cluster = self.marklogic.cluster()
cluster.couple(altcluster)
print("Finished")
def ml_init(self, hostname):
print("{0}: initialize host...".format(hostname))
try:
host = MarkLogic.instance_init(hostname)
except UnauthorizedAPIRequest:
# Assume that this happened because the host is already initialized
host = Host(hostname)
return host.just_initialized()
def ml_join(self, boothost, hostname):
print("{0}: join cluster with {1}...".format(hostname, boothost))
cluster = self.marklogic.cluster()
host = Host(hostname)
cluster.add_host(host)
def ml_security(self, hostname):
print("{0}: initialize security...".format(hostname))
MarkLogic.instance_admin(hostname, self.realm, self.adminuser,
self.adminpass)
class Docker:
def __init__(self):
self.marklogic = None
self.uname = pwd.getpwuid(os.getuid()).pw_name
self.adminuser = "******"
self.adminpass = "******"
self.realm = "public"
self.imatch = "ml[0-9]"
self.bootimage = None
self.localimage = False
self.count = None
self.couple = None
self.couple_user = None
self.couple_pass = None
self.name = None
self.blacklist_file = "/tmp/{0}.docker.skip".format(self.uname)
self.blacklist = {}
self.container_list = []
self.cluster_list = []
self.containers = {}
self.localip = None
self.ipaddr = {}
self.hostname = {}
def set_credentials(self, user, password):
self.adminuser = user
self.adminpass = password
def set_realm(self, realm):
self.realm = realm
def set_image_match(self, match):
self.imatch = match
def set_boot_image(self, image):
self.bootimage = image
self.localimage = (image == "localhost")
def set_count(self, count):
self.count = count
def set_couple(self, couple):
self.couple = couple
def set_couple_credentials(self, user, password):
self.couple_user = user
self.couple_pass = password
def set_name(self, name):
self.name = name
def setup_cluster(self):
if self.couple is not None and self.couple_pass is None:
self.couple_pass = self.adminpass
self.couple_user = self.adminuser
self.load_blacklist()
self.find_containers()
if not self.container_list:
print("There must be at least one unused container running.")
sys.exit(1)
if self.localimage:
ps = os.popen("docker exec " + self.container_list[0] +
" ip route")
for line in ps.readlines():
match = re.match("^default via (\S+)", line)
if match:
self.localip = match.group(1)
if self.localip is None:
print("Cannot find IP address of localhost!?")
sys.exit(1)
# Find the bootstrap image
if self.localimage:
pass
else:
self.bootimage = self.pick_image(self.bootimage)
if self.bootimage in self.container_list:
self.container_list.remove(self.bootimage)
self.cluster_list = self.pick_containers()
self.display_info()
#sys.exit(1)
# Initialize the bootstrap image, if necessary
if self.localimage:
bootip = self.localip
else:
bootip = self.ipaddr[self.bootimage]
if self.ml_init(bootip, self.bootimage):
self.ml_security(bootip, self.bootimage)
conn = Connection(bootip, HTTPDigestAuth(self.adminuser,
self.adminpass))
self.marklogic = MarkLogic(conn)
for container in self.cluster_list:
if container == self.bootimage:
continue
ip = self.ipaddr[container]
self.ml_init(ip, container)
self.ml_join(bootip, ip)
if self.name is not None:
print("{0}: rename cluster...".format(bootip))
cluster = self.marklogic.cluster()
cluster.set_cluster_name(self.name)
cluster.update()
if self.couple is not None:
for couple in self.couple:
print("{0}: couple with {1}...".format(bootip, couple))
altconn = Connection(
couple, HTTPDigestAuth(self.couple_user, self.couple_pass))
altml = MarkLogic(altconn)
altcluster = altml.cluster()
cluster = self.marklogic.cluster()
cluster.couple(altcluster)
print("Finished")
def ml_init(self, ip, container):
if container in self.hostname:
hostname = self.hostname[container]
else:
hostname = container
print("{0}: initialize host {1}...".format(ip, hostname))
try:
host = MarkLogic.instance_init(ip)
except UnauthorizedAPIRequest:
# Assume that this happened because the host is already initialized
host = Host(ip)
self.blacklist[container] = "used"
self.save_blacklist()
return host.just_initialized()
def ml_join(self, bootip, ip):
print("{0}: join cluster with {1}...".format(ip, bootip))
cluster = self.marklogic.cluster()
host = Host(ip)
cluster.add_host(host)
def ml_security(self, ip, container):
print("{0}: initialize security...".format(ip))
MarkLogic.instance_admin(ip, self.realm, self.adminuser,
self.adminpass)
self.blacklist[container] = "boot"
self.save_blacklist()
def pick_image(self, name):
if name is None:
for container in self.container_list:
if not container in self.blacklist:
return container
print("Cannot find unused container")
sys.exit(1)
for key in self.hostname:
if name == self.hostname[key]:
return key
count = 0
container = None
for key in self.ipaddr:
if name == self.ipaddr[key]:
return key
if re.match("^" + name, key):
count += 1
container = key
if count == 1:
return container
if count > 1:
print("Ambiguous container id:", name)
sys.exit(1)
for key in self.containers:
image = self.containers[key]
if (key == self.bootimage or re.match(self.bootimage, image)):
return key
print("Cannot find container \"{0}\".".format(name))
sys.exit(1)
def pick_containers(self):
if self.count is None:
count = -1
else:
count = self.count - 1
cset = set()
if not self.localimage:
cset.add(self.bootimage)
for container in self.container_list:
if container == self.bootimage or container in self.blacklist:
continue
if count == 0:
continue
cset.add(container)
count -= 1
containers = []
for key in cset:
containers.append(key)
return containers
def load_blacklist(self):
try:
f = open(self.blacklist_file, "r")
for line in f.readlines():
(container, flag) = line.strip().split(" ")
self.blacklist[container] = flag
f.close()
except FileNotFoundError:
pass
def save_blacklist(self):
f = open(self.blacklist_file, "w")
for key in self.blacklist:
if self.blacklist[key] != "gone":
f.write("{0} {1}\n".format(key, self.blacklist[key]))
f.close()
def find_containers(self):
self.load_blacklist()
ps = os.popen("docker ps")
for line in ps.readlines():
match = re.match("([0-9a-f]+)\s+(\S+)", line)
if match:
container = match.group(1)
image = match.group(2)
ipx = os.popen("docker inspect {0}".format(container))
ip = None
hostname = None
for line in ipx.readlines():
match = re.match('\s+"IPAddress": "([^"]+)"', line)
if match:
ip = match.group(1)
match = re.match('\s+"Hostname": "([^"]+)"', line)
if match:
hostname = match.group(1)
if ip is None:
print("Cannot read IP address of container {0}".format(
container))
else:
self.ipaddr[container] = ip
if hostname is None:
print("Cannot read hostname of container {0}".format(
container))
else:
self.hostname[container] = hostname
if re.match(self.imatch, image):
self.container_list = [container] + self.container_list
self.containers[container] = image
for container in self.blacklist:
if not container in self.containers:
self.blacklist[container] = "gone"
def display_info(self):
# Display information about docker containers
# Sigh. Report formatting is such a drag...
dinfo = []
ps = os.popen("docker ps")
for line in ps.readlines():
match = re.match("([0-9a-f]+)\s+(\S+)", line)
if match:
container = match.group(1)
image = match.group(2)
ip = self.ipaddr[container]
hostname = self.hostname[container]
attr = []
if container in self.blacklist:
attr.append("used")
if self.blacklist[container] == "boot":
attr.append("boot")
else:
if not container in self.cluster_list:
attr.append("skip")
else:
if re.match(self.imatch, image):
attr.append("*")
if container == self.bootimage:
attr.append("boot")
else:
attr.append("!~" + self.imatch)
data = {
"container": container,
"image": image,
"ipaddr": ip,
"hostname": hostname,
"flags": ", ".join(attr)
}
dinfo = [data] + dinfo
headers = {
"container": "Container",
"image": "Image",
"hostname": "Hostname",
"ipaddr": "IP Addr",
"flags": "Flags"
}
maxlen = {}
for key in dinfo[0]:
flen = len(headers[key])
for data in dinfo:
if len(data[key]) > flen:
flen = len(data[key])
maxlen[key] = flen
fstr = "%-{0}s %-{1}s %-{2}s %-{3}s %s".format(
maxlen['container'], maxlen['image'], maxlen['hostname'],
maxlen['ipaddr'])
print(fstr %
(headers['container'], headers['image'], headers['hostname'],
headers['ipaddr'], headers['flags']))
for data in dinfo:
print(fstr % (data['container'], data['image'], data['hostname'],
data['ipaddr'], data['flags']))
def ml_security(self, hostname):
print("{0}: initialize security...".format(hostname))
MarkLogic.instance_admin(hostname, self.realm, self.adminuser, self.adminpass)
class CreateForests:
def __init__(self):
self.marklogic = None
self.host = "localhost"
self.hosts = None
self.prefix = "F"
self.forests = 2
self.start = 1
self.failover = "none"
self.failhost = None
self.adminuser = "******"
self.adminpass = "******"
self.data_dir = None
self.large_data_dir = None
self.fast_data_dir = None
self.database = None
self.dry_run = False
# TODO: better checking of argument types
def set_rest_host(self, host):
self.host = host
return self
def set_hosts(self, hosts):
if hosts is None:
self.hosts = None
else:
if isinstance(hosts, list):
self.hosts = hosts
else:
self.hosts = [ hosts ]
return self
def set_prefix(self, prefix):
self.prefix = prefix
return self
def set_forest_count(self, forests):
self.forests = int(forests)
return self
def set_start_number(self, start):
self.start = start
return self
def set_failover(self, failover):
self.failover = failover
return self
def set_failover_host(self, failhost):
self.failhost = failhost
return self
def set_user(self, adminuser):
self.adminuser = adminuser
return self
def set_pass(self, adminpass):
self.adminpass = adminpass
return self
def set_data_dir(self, data_dir):
self.data_dir = data_dir
return self
def set_large_data_dir(self, large_data_dir):
self.large_data_dir = large_data_dir
return self
def set_fast_data_dir(self, fast_data_dir):
self.fast_data_dir = fast_data_dir
return self
def set_database(self, database):
self.database = database
return self
def set_dry_run(self, dry_run):
self.dry_run = dry_run
return self
def create_forests(self):
conn = Connection(self.host,
HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
if self.failhost is None and self.failover != "none":
print("Invalid configuration, specify failover-host for failover:",
self.failover)
sys.exit(1)
if self.hosts is None:
host_names = self.marklogic.hosts()
else:
host_names = self.hosts
exists = []
host_index = 0
for host_name in host_names:
host_index += 1
for count in range(1,self.forests + 1):
name = self.prefix + "-" + str(host_index) + "-" + str(count)
forest = Forest(name, host_name, conn)
if forest.exists():
exists.append(host_name + ":" + name)
if len(exists) != 0:
print("Abort: forest(s) already exist:")
for f in exists:
print(" ", f)
sys.exit(1)
host_index = 0
for host_name in host_names:
host_index += 1
for count in range(self.start,self.start + self.forests):
name = self.prefix + "-" + str(host_index) + "-" + str(count)
forest = Forest(name, host_name, conn)
if self.data_dir is not None:
forest.set_data_directory(self.data_dir)
if self.large_data_dir is not None:
forest.set_large_data_directory(self.large_data_dir)
if self.fast_data_dir is not None:
forest.set_fast_data_directory(self.fast_data_dir)
if self.failhost is not None:
forest.set_failover(self.failover)
forest.set_failover_host_names(self.failhost)
if self.database is not None:
forest.set_database(self.database)
print("Create forest " + name + " on " + host_name)
if self.dry_run:
print(json.dumps(forest.marshal(), sort_keys=True, indent=2))
else:
forest.create()
print("Finished")
class App:
def __init__(self):
self.marklogic = None
self.uname = pwd.getpwuid(os.getuid()).pw_name
self.adminuser = "******"
self.adminpass = "******"
self.realm = "public"
self.host = None
self.boothost = None
self.couple = None
self.couple_user = None
self.couple_pass = None
self.name = None
def set_credentials(self, user, password):
self.adminuser = user
self.adminpass = password
def set_realm(self, realm):
self.realm = realm
def set_boot_host(self, host):
self.boothost = host
def set_host(self, name):
self.host = name
def set_couple(self, couple):
self.couple = couple
def set_couple_credentials(self, user, password):
self.couple_user = user
self.couple_pass = password
def set_name(self, name):
self.name = name
def setup_cluster(self):
if self.couple is not None and self.couple_pass is None:
self.couple_pass = self.adminpass
self.couple_user = self.adminuser
if self.boothost is None:
self.boothost = self.host[0]
self.host.remove(self.boothost)
if self.ml_init(self.boothost):
self.ml_security(self.boothost)
conn = Connection(self.boothost, HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
for hostname in self.host:
self.ml_init(hostname)
self.ml_join(self.boothost, hostname)
if self.name is not None:
print("{0}: rename cluster...".format(self.boothost))
cluster = self.marklogic.cluster()
cluster.set_cluster_name(self.name)
cluster.update()
if self.couple is not None:
for couple in self.couple:
print("{0}: couple with {1}...".format(self.boothost, couple))
altconn = Connection(couple,
HTTPDigestAuth(self.couple_user,
self.couple_pass))
altml = MarkLogic(altconn)
altcluster = altml.cluster()
cluster = self.marklogic.cluster()
cluster.couple(altcluster)
print("Finished")
def ml_init(self, hostname):
print("{0}: initialize host...".format(hostname))
try:
host = MarkLogic.instance_init(hostname)
except UnauthorizedAPIRequest:
# Assume that this happened because the host is already initialized
host = Host(hostname)
return host.just_initialized()
def ml_join(self, boothost, hostname):
print("{0}: join cluster with {1}...".format(hostname, boothost))
cluster = self.marklogic.cluster()
host = Host(hostname)
cluster.add_host(host)
def ml_security(self, hostname):
print("{0}: initialize security...".format(hostname))
MarkLogic.instance_admin(hostname, self.realm, self.adminuser, self.adminpass)
def setup_cluster(self):
if self.couple is not None and self.couple_pass is None:
self.couple_pass = self.adminpass
self.couple_user = self.adminuser
self.load_blacklist()
self.find_containers()
if not self.container_list:
print("There must be at least one unused container running.")
sys.exit(1)
if self.localimage:
ps = os.popen("docker exec " + self.container_list[0] +
" ip route")
for line in ps.readlines():
match = re.match("^default via (\S+)", line)
if match:
self.localip = match.group(1)
if self.localip is None:
print("Cannot find IP address of localhost!?")
sys.exit(1)
# Find the bootstrap image
if self.localimage:
pass
else:
self.bootimage = self.pick_image(self.bootimage)
if self.bootimage in self.container_list:
self.container_list.remove(self.bootimage)
self.cluster_list = self.pick_containers()
self.display_info()
#sys.exit(1)
# Initialize the bootstrap image, if necessary
if self.localimage:
bootip = self.localip
else:
bootip = self.ipaddr[self.bootimage]
if self.ml_init(bootip, self.bootimage):
self.ml_security(bootip, self.bootimage)
conn = Connection(bootip, HTTPDigestAuth(self.adminuser,
self.adminpass))
self.marklogic = MarkLogic(conn)
for container in self.cluster_list:
if container == self.bootimage:
continue
ip = self.ipaddr[container]
self.ml_init(ip, container)
self.ml_join(bootip, ip)
if self.name is not None:
print("{0}: rename cluster...".format(bootip))
cluster = self.marklogic.cluster()
cluster.set_cluster_name(self.name)
cluster.update()
if self.couple is not None:
for couple in self.couple:
print("{0}: couple with {1}...".format(bootip, couple))
altconn = Connection(
couple, HTTPDigestAuth(self.couple_user, self.couple_pass))
altml = MarkLogic(altconn)
altcluster = altml.cluster()
cluster = self.marklogic.cluster()
cluster.couple(altcluster)
print("Finished")
def backup_databases(self):
conn = Connection(self.host,
HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
actual_databases = self.marklogic.databases()
if self.databases is None:
self.databases = actual_databases
if self.backup_root is None:
raise UnsupportedOperation("You must specify the backup root.")
if self.max_parallel <= 0:
self.max_parallel = 1
for dbname in self.databases:
if not dbname in actual_databases:
raise UnsupportedOperation("Database does not exist: {0}"
.format(dbname))
maxp = self.max_parallel
running = 0
done = False
queue = self.databases
status_list = {}
min_wait = 5
max_wait = 30
wait_incr = 5
wait = min_wait
while not done:
done = True
while len(queue) > 0 and running < maxp:
dbname = queue.pop(0)
running += 1
done = False
print("Backing up {0}".format(dbname))
if not self.dry_run:
db = self.marklogic.database(dbname)
bkp = db.backup(self.backup_root + dbname, connection=conn)
status_list[dbname] = bkp
response = bkp.status()
if response['status'] != 'in-progress':
print("{0}: {1}".format(dbname, response['status']))
if self.dry_run:
if running > 0 or len(queue) > 0:
print("{0} backups in dry-run; {1} in queue..."
.format(running, len(queue)))
running = 0
else:
if len(status_list) > 0:
new_list = {}
for dbname in status_list:
bkp = status_list[dbname]
response = bkp.status()
print("{0}: {1}".format(dbname, response['status']))
if response['status'] == 'in-progress':
done = False
new_list[dbname] = bkp
else:
running -= 1
wait = min_wait
done = done and len(queue) == 0
if not done:
status_list = new_list
if running < maxp and len(queue) != 0:
print("Running: {0} backups running; {1} in queue..."
.format(running, len(queue)))
wait = min_wait
print("")
else:
print("Waiting {0}s: {1} backups running; {2} in queue..."
.format(wait, running, len(queue)))
time.sleep(wait)
if wait < max_wait:
wait += wait_incr
print("")
class InitServer:
def __init__(self):
pass
#logging.basicConfig(level=logging.INFO)
parser = argparse.ArgumentParser()
parser.add_argument("--host", action='store', default="localhost",
help="Management API host")
parser.add_argument("--username", action='store', default="admin",
help="User name")
parser.add_argument("--password", action='store', default="admin",
help="Password")
parser.add_argument("--wallet", action='store', default="admin",
help="Wallet password")
parser.add_argument('--debug', action='store_true',
help='Enable debug logging')
args = parser.parse_args()
if args.debug:
logging.basicConfig(level=logging.WARNING)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("marklogic").setLevel(logging.DEBUG)
print("Initialize host {}".format(args.host))
MarkLogic.instance_init(args.host)
print("Initialize admin {}".format(args.host))
MarkLogic.instance_admin(args.host, "public", args.username, args.password, args.wallet)
print("finished")
def __init__(self, argv):
super().__init__(argv)
self.connection = Connection(tc.hostname,
HTTPDigestAuth(tc.admin, tc.password))
self.marklogic = MarkLogic(self.connection)
class Docker:
def __init__(self):
self.marklogic = None
self.uname=pwd.getpwuid(os.getuid()).pw_name
self.adminuser="******"
self.adminpass="******"
self.realm="public"
self.imatch="ml[0-9]"
self.bootimage=None
self.localimage=False
self.count=None
self.couple=None
self.couple_user=None
self.couple_pass=None
self.name=None
self.blacklist_file="/tmp/{0}.docker.skip".format(self.uname)
self.blacklist={}
self.container_list=[]
self.cluster_list=[]
self.containers={}
self.localip=None
self.ipaddr={}
self.hostname={}
def set_credentials(self,user,password):
self.adminuser = user
self.adminpass = password
def set_realm(self,realm):
self.realm = realm
def set_image_match(self,match):
self.imatch = match
def set_boot_image(self,image):
self.bootimage = image
self.localimage = (image == "localhost")
def set_count(self,count):
self.count = count
def set_couple(self,couple):
self.couple = couple
def set_couple_credentials(self,user,password):
self.couple_user = user
self.couple_pass = password
def set_name(self,name):
self.name = name
def setup_cluster(self):
if self.couple is not None and self.couple_pass is None:
self.couple_pass = self.adminpass
self.couple_user = self.adminuser
self.load_blacklist()
self.find_containers()
if not self.container_list:
print("There must be at least one unused container running.")
sys.exit(1)
if self.localimage:
ps = os.popen("docker exec " + self.container_list[0] + " ip route")
for line in ps.readlines():
match = re.match("^default via (\S+)", line)
if match:
self.localip = match.group(1)
if self.localip is None:
print("Cannot find IP address of localhost!?")
sys.exit(1)
# Find the bootstrap image
if self.localimage:
pass
else:
self.bootimage = self.pick_image(self.bootimage)
self.container_list.remove(self.bootimage)
self.cluster_list = self.pick_containers()
self.display_info()
#sys.exit(1)
# Initialize the bootstrap image, if necessary
if self.localimage:
bootip = self.localip
else:
bootip = self.ipaddr[self.bootimage]
if self.ml_init(bootip, self.bootimage):
self.ml_security(bootip, self.bootimage)
conn = Connection(bootip, HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
for container in self.cluster_list:
if container == self.bootimage:
continue
ip = self.ipaddr[container]
self.ml_init(ip, container)
self.ml_join(bootip, ip)
if self.name is not None:
print("{0}: rename cluster...".format(bootip))
cluster = self.marklogic.cluster()
cluster.set_cluster_name(self.name)
cluster.update()
if self.couple is not None:
for couple in self.couple:
print("{0}: couple with {1}...".format(bootip,couple))
altconn = Connection(couple,
HTTPDigestAuth(self.couple_user,
self.couple_pass))
altml = MarkLogic(altconn)
altcluster = altml.cluster()
cluster = self.marklogic.cluster()
cluster.couple(altcluster)
print("Finished")
def ml_init(self, ip, container):
if container in self.hostname:
hostname = self.hostname[container]
else:
hostname = container
print("{0}: initialize host {1}...".format(ip,hostname))
try:
host = MarkLogic.instance_init(ip)
except UnauthorizedAPIRequest:
# Assume that this happened because the host is already initialized
host = Host(ip)
self.blacklist[container] = "used"
self.save_blacklist()
return host.just_initialized()
def ml_join(self, bootip, ip):
print("{0}: join cluster with {1}...".format(ip,bootip))
cluster = self.marklogic.cluster()
host = Host(ip)
cluster.add_host(host)
def ml_security(self, ip, container):
print("{0}: initialize security...".format(ip))
MarkLogic.instance_admin(ip,self.realm,self.adminuser,self.adminpass)
self.blacklist[container] = "boot"
self.save_blacklist()
def pick_image(self, name):
if name is None:
for container in self.container_list:
if not container in self.blacklist:
return container
print("Cannot find unused container")
sys.exit(1)
for key in self.hostname:
if name == self.hostname[key]:
return key
count = 0
container = None
for key in self.ipaddr:
if name == self.ipaddr[key]:
return key
if re.match("^"+name, key):
count += 1
container = key
if count == 1:
return container
if count > 1:
print("Ambiguous container id:", name)
sys.exit(1)
for key in self.containers:
image = self.containers[key]
if (key == self.bootimage or re.match(self.bootimage, image)):
return key
print("Cannot find container \"{0}\".".format(name))
sys.exit(1)
def pick_containers(self):
if self.count is None:
count = -1
else:
count = self.count - 1
cset = set()
if not self.localimage:
cset.add(self.bootimage)
for container in self.container_list:
if container == self.bootimage or container in self.blacklist:
continue
if count == 0:
continue
cset.add(container)
count -= 1
containers = []
for key in cset:
containers.append(key)
return containers
def load_blacklist(self):
try:
f = open(self.blacklist_file, "r")
for line in f.readlines():
(container, flag) = line.strip().split(" ")
self.blacklist[container] = flag
f.close()
except FileNotFoundError:
pass
def save_blacklist(self):
f = open(self.blacklist_file, "w")
for key in self.blacklist:
if self.blacklist[key] != "gone":
f.write("{0} {1}\n".format(key,self.blacklist[key]))
f.close()
def find_containers(self):
self.load_blacklist()
ps = os.popen("docker ps")
for line in ps.readlines():
match = re.match("([0-9a-f]+)\s+(\S+)", line)
if match:
container = match.group(1)
image = match.group(2)
ipx = os.popen("docker inspect {0}".format(container))
ip = None
hostname = None
for line in ipx.readlines():
match = re.match('\s+"IPAddress": "([^"]+)"', line)
if match:
ip = match.group(1)
match = re.match('\s+"Hostname": "([^"]+)"', line)
if match:
hostname = match.group(1)
if ip is None:
print("Cannot read IP address of container {0}"
.format(container))
else:
self.ipaddr[container] = ip
if hostname is None:
print("Cannot read hostname of container {0}"
.format(container))
else:
self.hostname[container] = hostname
if re.match(self.imatch, image):
self.container_list = [container] + self.container_list
self.containers[container] = image
for container in self.blacklist:
if not container in self.containers:
self.blacklist[container] = "gone"
def display_info(self):
# Display information about docker containers
# Sigh. Report formatting is such a drag...
dinfo = []
ps = os.popen("docker ps")
for line in ps.readlines():
match = re.match("([0-9a-f]+)\s+(\S+)", line)
if match:
container = match.group(1)
image = match.group(2)
ip = self.ipaddr[container]
hostname = self.hostname[container]
attr = []
if container in self.blacklist:
attr.append("used")
if self.blacklist[container] == "boot":
attr.append("boot")
else:
if not container in self.cluster_list:
attr.append("skip")
else:
if re.match(self.imatch, image):
attr.append("*")
if container == self.bootimage:
attr.append("boot")
else:
attr.append("!~"+self.imatch)
data = {"container": container,
"image": image,
"ipaddr": ip,
"hostname": hostname,
"flags": ", ".join(attr)}
dinfo = [data] + dinfo
headers = {"container": "Container",
"image": "Image",
"hostname": "Hostname",
"ipaddr": "IP Addr",
"flags": "Flags"}
maxlen = {}
for key in dinfo[0]:
flen = len(headers[key])
for data in dinfo:
if len(data[key]) > flen:
flen = len(data[key])
maxlen[key] = flen
fstr = "%-{0}s %-{1}s %-{2}s %-{3}s %s".format(
maxlen['container'], maxlen['image'],
maxlen['hostname'], maxlen['ipaddr'])
print(fstr % (headers['container'], headers['image'],
headers['hostname'], headers['ipaddr'],
headers['flags']))
for data in dinfo:
print(fstr % (data['container'], data['image'],
data['hostname'], data['ipaddr'], data['flags']))
def ml_security(self, hostname):
print("{0}: initialize security...".format(hostname))
MarkLogic.instance_admin(hostname, self.realm, self.adminuser,
self.adminpass)
def ml_security(self, ip, container):
print("{0}: initialize security...".format(ip))
MarkLogic.instance_admin(ip,self.realm,self.adminuser,self.adminpass)
self.blacklist[container] = "boot"
self.save_blacklist()
def ml_security(self, ip, container):
print("{0}: initialize security...".format(ip))
MarkLogic.instance_admin(ip, self.realm, self.adminuser,
self.adminpass)
self.blacklist[container] = "boot"
self.save_blacklist()
def restore_databases(self):
conn = Connection(self.host,
HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
actual_databases = self.marklogic.databases()
if self.databases is None:
self.databases = actual_databases
if self.restore_root is None:
raise UnsupportedOperation("You must specify the restore root.")
if self.max_parallel <= 0:
self.max_parallel = 1
for dbname in self.databases:
if not dbname in actual_databases:
raise UnsupportedOperation("Database does not exist: {0}"
.format(dbname))
maxp = self.max_parallel
running = 0
done = False
queue = self.databases
status_list = {}
min_wait = 5
max_wait = 30
wait_incr = 5
wait = min_wait
while not done:
done = True
while len(queue) > 0 and running < maxp:
dbname = queue.pop(0)
running += 1
done = False
print("Restoring {0}".format(dbname))
if not self.dry_run:
db = self.marklogic.database(dbname)
rst = db.restore(self.restore_root + dbname, connection=conn)
status_list[dbname] = rst
response = rst.status()
for forest in response['forest']:
if forest['status'] != 'in-progress':
print("Forest {0}: {1}"
.format(forest['forest-name'],
forest['status']))
if self.dry_run:
if running > 0 or len(queue) > 0:
print("{0} restores in dry-run; {1} in queue..."
.format(running, len(queue)))
running = 0
else:
if len(status_list) > 0:
new_list = {}
for dbname in status_list:
rst = status_list[dbname]
response = rst.status()
fdone = True
for forest in response['forest']:
print("Forest {0}: {1}"
.format(forest['forest-name'],
forest['status']))
if forest['status'] == 'in-progress':
fdone = False
if not fdone:
done = False
new_list[dbname] = rst
else:
running -= 1
wait = min_wait
done = done and len(queue) == 0
if not done:
status_list = new_list
if running < maxp and len(queue) != 0:
print("Running: {0} restores running; {1} in queue..."
.format(running, len(queue)))
wait = min_wait
print("")
else:
print("Waiting {0}s: {1} restores running; {2} in queue..."
.format(wait, running, len(queue)))
time.sleep(wait)
if wait < max_wait:
wait += wait_incr
print("")
class RestoreDatabases:
def __init__(self):
self.marklogic = None
self.host = "localhost"
self.adminuser = "******"
self.adminpass = "******"
self.databases = None
self.restore_root = None
self.journal_arch = False
self.lag_limit = 30
self.incremental = False
self.max_parallel = 5
self.dry_run = False
# TODO: better checking of argument types
def set_host(self, host):
self.host = host
return self
def set_user(self, adminuser):
self.adminuser = adminuser
return self
def set_pass(self, adminpass):
self.adminpass = adminpass
return self
def set_databases(self, databases):
self.databases = databases
def set_restore_root(self, directory):
if directory is not None:
self.restore_root = directory
if not self.restore_root.endswith("/"):
self.restore_root += "/"
return self
def set_journal_archiving(self, archiving):
self.journal_arch = archiving
return self
def set_lag_limit(self, limit):
self.lag_limit = limit
return self
def set_incremental(self, incremental):
self.incremental = incremental
return self
def set_max_parallel(self, max_parallel):
self.max_parallel = max_parallel
return self
def set_dry_run(self, dry_run):
self.dry_run = dry_run
return self
def restore_databases(self):
conn = Connection(self.host,
HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
actual_databases = self.marklogic.databases()
if self.databases is None:
self.databases = actual_databases
if self.restore_root is None:
raise UnsupportedOperation("You must specify the restore root.")
if self.max_parallel <= 0:
self.max_parallel = 1
for dbname in self.databases:
if not dbname in actual_databases:
raise UnsupportedOperation("Database does not exist: {0}"
.format(dbname))
maxp = self.max_parallel
running = 0
done = False
queue = self.databases
status_list = {}
min_wait = 5
max_wait = 30
wait_incr = 5
wait = min_wait
while not done:
done = True
while len(queue) > 0 and running < maxp:
dbname = queue.pop(0)
running += 1
done = False
print("Restoring {0}".format(dbname))
if not self.dry_run:
db = self.marklogic.database(dbname)
rst = db.restore(self.restore_root + dbname, connection=conn)
status_list[dbname] = rst
response = rst.status()
for forest in response['forest']:
if forest['status'] != 'in-progress':
print("Forest {0}: {1}"
.format(forest['forest-name'],
forest['status']))
if self.dry_run:
if running > 0 or len(queue) > 0:
print("{0} restores in dry-run; {1} in queue..."
.format(running, len(queue)))
running = 0
else:
if len(status_list) > 0:
new_list = {}
for dbname in status_list:
rst = status_list[dbname]
response = rst.status()
fdone = True
for forest in response['forest']:
print("Forest {0}: {1}"
.format(forest['forest-name'],
forest['status']))
if forest['status'] == 'in-progress':
fdone = False
if not fdone:
done = False
new_list[dbname] = rst
else:
running -= 1
wait = min_wait
done = done and len(queue) == 0
if not done:
status_list = new_list
if running < maxp and len(queue) != 0:
print("Running: {0} restores running; {1} in queue..."
.format(running, len(queue)))
wait = min_wait
print("")
else:
print("Waiting {0}s: {1} restores running; {2} in queue..."
.format(wait, running, len(queue)))
time.sleep(wait)
if wait < max_wait:
wait += wait_incr
print("")
class BackupDatabases:
def __init__(self):
self.marklogic = None
self.host = "localhost"
self.adminuser = "******"
self.adminpass = "******"
self.databases = None
self.backup_root = None
self.journal_arch = False
self.lag_limit = 30
self.incremental = False
self.max_parallel = 5
self.dry_run = False
# TODO: better checking of argument types
def set_host(self, host):
self.host = host
return self
def set_user(self, adminuser):
self.adminuser = adminuser
return self
def set_pass(self, adminpass):
self.adminpass = adminpass
return self
def set_databases(self, databases):
self.databases = databases
def set_backup_root(self, directory):
if directory is not None:
self.backup_root = directory
if not self.backup_root.endswith("/"):
self.backup_root += "/"
return self
def set_journal_archiving(self, archiving):
self.journal_arch = archiving
return self
def set_lag_limit(self, limit):
self.lag_limit = limit
return self
def set_incremental(self, incremental):
self.incremental = incremental
return self
def set_max_parallel(self, max_parallel):
self.max_parallel = max_parallel
return self
def set_dry_run(self, dry_run):
self.dry_run = dry_run
return self
def backup_databases(self):
conn = Connection(self.host,
HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
actual_databases = self.marklogic.databases()
if self.databases is None:
self.databases = actual_databases
if self.backup_root is None:
raise UnsupportedOperation("You must specify the backup root.")
if self.max_parallel <= 0:
self.max_parallel = 1
for dbname in self.databases:
if not dbname in actual_databases:
raise UnsupportedOperation("Database does not exist: {0}"
.format(dbname))
maxp = self.max_parallel
running = 0
done = False
queue = self.databases
status_list = {}
min_wait = 5
max_wait = 30
wait_incr = 5
wait = min_wait
while not done:
done = True
while len(queue) > 0 and running < maxp:
dbname = queue.pop(0)
running += 1
done = False
print("Backing up {0}".format(dbname))
if not self.dry_run:
db = self.marklogic.database(dbname)
bkp = db.backup(self.backup_root + dbname, connection=conn)
status_list[dbname] = bkp
response = bkp.status()
if response['status'] != 'in-progress':
print("{0}: {1}".format(dbname, response['status']))
if self.dry_run:
if running > 0 or len(queue) > 0:
print("{0} backups in dry-run; {1} in queue..."
.format(running, len(queue)))
running = 0
else:
if len(status_list) > 0:
new_list = {}
for dbname in status_list:
bkp = status_list[dbname]
response = bkp.status()
print("{0}: {1}".format(dbname, response['status']))
if response['status'] == 'in-progress':
done = False
new_list[dbname] = bkp
else:
running -= 1
wait = min_wait
done = done and len(queue) == 0
if not done:
status_list = new_list
if running < maxp and len(queue) != 0:
print("Running: {0} backups running; {1} in queue..."
.format(running, len(queue)))
wait = min_wait
print("")
else:
print("Waiting {0}s: {1} backups running; {2} in queue..."
.format(wait, running, len(queue)))
time.sleep(wait)
if wait < max_wait:
wait += wait_incr
print("")
