def instance_admin(cls,host,realm,admin,password):
"""
Initializes the security database of a newly initialized server.
:param host: The name or IP address of the host to initialize
:param realm: The security realm to install
:param admin: The name of the admin user
:param password: The password of the admin user
"""
conn = Connection(host, None)
payload = {
'admin-username': admin,
'admin-password': password,
'realm': realm
}
uri = "{0}://{1}:8001/admin/v1/instance-admin".format(
conn.protocol, conn.host)
logger = logging.getLogger("marklogic")
logger.debug("Initializing security for {0}".format(host))
# N.B. Can't use conn.post here because we don't need auth yet
response = requests.post(uri, json=payload,
headers={'content-type': 'application/json',
'accept': 'application/json'})
if response.status_code != 202:
raise UnexpectedManagementAPIResponse(response.text)
# From now on connections require auth...
conn = Connection(host, HTTPDigestAuth(admin, password))
data = json.loads(response.text)
conn.wait_for_restart(data["restart"]["last-startup"][0]["value"])
def couple_cluster(self):
conn = Connection(self.host,
HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
if self.couple is not None:
for couple in self.couple:
print("{0}: couple with {1}...".format(self.host, couple))
altconn = Connection(
couple, HTTPDigestAuth(self.couple_user, self.couple_pass))
altml = MarkLogic(altconn)
altcluster = altml.cluster()
cluster = self.marklogic.cluster()
cluster.couple(altcluster)
print("Finished")
def __init__(self, *args, **kwargs):
super(MLConfig,self).__init__(*args, **kwargs)
config = { "hostname": "localhost", \
"username": "******", \
"password": "******", \
"protocol": "http", \
"port": 8000, \
"management-port": 8002, \
"root": "manage", \
"version": "v2", \
"client-version": "v1" }
try:
data_file = open("mlconfig.json").read()
data = json.loads(data_file)
for key in data:
config[key] = data[key]
except FileNotFoundError:
pass
self.auth = HTTPDigestAuth(config["username"], config["password"])
self.connection = Connection(config["hostname"], self.auth, \
protocol=config["protocol"], \
port=config["port"], \
management_port=config["management-port"], \
root=config["root"], \
version=config["version"], \
client_version=config["client-version"])
def test_create_single_detailed_forest(self):
"""
Test the following scenario:
The database is given a forest object. It should create a forest with
the given name. That forest should match the features of the datailed
forest.
"""
conn = Connection(tc.hostname, HTTPDigestAuth(tc.admin, tc.password))
hosts = Host.list(conn)
db = Database("detailed-forest-create-test-db", hosts[0])
forest = Forest("detailed-forest-create-forest1", host=hosts[0])
forest.set_large_data_directory(ds.large_data_directory)
db.set_forest_names([forest.forest_name()])
db.create(conn)
forest = Forest.lookup(conn, "detailed-forest-create-forest1")
try:
self.assertEqual("detailed-forest-create-forest1",
forest.forest_name())
#this isn't in the properties...oddly.
#self.assertEqual(ds.large_data_directory, forest.large_data_directory())
finally:
db.delete(connection=conn)
def test_simple_create(self):
"""
TODO: The hostname should come from the server's hostname
Test the basic create function. Creates a database and then check to see that it
exists by getting the database configuration from the server. It then destroys
the database.
:return: None
"""
conn = Connection(tc.hostname, HTTPDigestAuth(tc.admin, tc.password))
hosts = Host.list(conn)
db = Database("test-db", hosts[0])
db.create(conn)
validate_db = Database.lookup(conn, "test-db")
try:
self.assertIsNotNone(validate_db)
self.assertEqual('test-db', validate_db.database_name())
finally:
validate_db.delete(connection=conn)
validate_db = Database.lookup(conn, "test-db")
self.assertIsNone(validate_db)
def test_create_simple_forests(self):
"""
Test the following scenario:
The database is given the names of two forests.
It should then create the two named forests.
"""
conn = Connection(tc.hostname, HTTPDigestAuth(tc.admin, tc.password))
hosts = Host.list(conn)
db = Database("simple-forest-create-test-db",
hosts[0],
connection=conn)
db.set_forest_names(
["simple-forest-create-forest1", "simple-forest-create-forest2"])
db.create()
db = Database.lookup(conn, "simple-forest-create-test-db")
try:
self.assertEqual(2, len(db.forest_names()))
self.assertIn("simple-forest-create-forest1", db.forest_names())
self.assertIn("simple-forest-create-forest2", db.forest_names())
finally:
db.delete(connection=conn)
def instance_init(cls, host):
"""
Performs first-time initialization of a newly installed server.
:param host: The name or IP address of the host to initialize
"""
conn = Connection(host, None)
uri = "{0}://{1}:8001/admin/v1/init".format(conn.protocol, conn.host)
logger = logging.getLogger("marklogic")
logger.debug("Initializing {0}".format(host))
# This call is a little odd; we special case the 400 error that
# occurs if the host has alreadya been initialized.
try:
response = conn.post(
uri, content_type='application/x-www-form-urlencoded')
except UnexpectedManagementAPIResponse:
response = conn.response
if response.status_code == 400:
err = json.loads(response.text)
if "errorResponse" in err:
if "messageCode" in err["errorResponse"]:
if err["errorResponse"][
"messageCode"] == "MANAGE-ALREADYINIT":
return Host(host)
raise
if response.status_code != 202:
raise UnexpectedManagementAPIResponse(response.text)
return Host(host)._set_just_initialized()
def join_cluster(self, cluster, cluster_connection=None):
if cluster_connection is None:
cluster_connection = cluster.connection
xml = self._get_server_config()
cfgzip = cluster._post_server_config(xml, cluster_connection)
connection = Connection(self.host_name(), cluster_connection.auth)
self._post_cluster_config(cfgzip, connection)
def test_list_databases(self):
conn = Connection(tc.hostname, HTTPDigestAuth(tc.admin, tc.password))
db_names = Database.list(conn)
self.assertGreater(len(db_names), 4)
self.assertTrue("Modules" in db_names)
self.assertTrue("Documents" in db_names)
def add_host(self, host, connection=None):
if connection is None:
connection = self.connection
if isinstance(host, str):
host = Host(host)
xml = host._get_server_config()
cfgzip = self._post_server_config(xml,connection)
host_connection = Connection(host.host_name(), connection.auth)
host._post_cluster_config(cfgzip,host_connection)
def load_data(self):
simpleapp = SimpleApplication(tc.appname, tc.port)
conn = Connection(tc.hostname, HTTPDigestAuth(tc.admin, tc.password))
hostname = Host.list(conn)[0]
exampleapp = simpleapp.create(conn, hostname)
loader = MLCPLoader()
loader.load_directory(conn,
exampleapp['content'],
"data",
collections=["example1"],
prefix="/test/data1")
def setup_cluster(self):
if self.couple is not None and self.couple_pass is None:
self.couple_pass = self.adminpass
self.couple_user = self.adminuser
if self.boothost is None:
self.boothost = self.host[0]
self.host.remove(self.boothost)
if self.ml_init(self.boothost):
self.ml_security(self.boothost)
conn = Connection(self.boothost,
HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
for hostname in self.host:
self.ml_init(hostname)
self.ml_join(self.boothost, hostname)
if self.name is not None:
print("{0}: rename cluster...".format(self.boothost))
cluster = self.marklogic.cluster()
cluster.set_cluster_name(self.name)
cluster.update()
if self.couple is not None:
for couple in self.couple:
print("{0}: couple with {1}...".format(self.boothost, couple))
altconn = Connection(
couple, HTTPDigestAuth(self.couple_user, self.couple_pass))
altml = MarkLogic(altconn)
altcluster = altml.cluster()
cluster = self.marklogic.cluster()
cluster.couple(altcluster)
print("Finished")
def _get_server_config(self):
"""
Obtain the server configuration. This is the data necessary for
the first part of the handshake necessary to join a host to a
cluster. The returned data is not intended for introspection.
:return: The config. This is always XML.
"""
connection = Connection(self.host_name(), None)
uri = "http://{0}:8001/admin/v1/server-config".format(connection.host)
response = connection.get(uri, accept="application/xml")
if response.status_code != 200:
raise UnexpectedManagementAPIResponse(response.text)
return response.text # this is always XML
def couple(self, args, config, connection):
cluster = LocalCluster(connection=connection)
cluster.read()
try:
username, password = re.split(":", args['couple_credentials'])
except ValueError:
print("--couple-credentials value must be 'user:password':",
args['couple_credentials'])
sys.exit(1)
altconn = Connection(args['host'], HTTPDigestAuth(username, password))
altcluster = LocalCluster(connection=altconn)
cluster.couple(altcluster,
connection=connection,
other_cluster_connection=altconn)
def connect(self, args):
try:
adminuser, adminpass = re.split(":", args['credentials'])
except ValueError:
print("--credentials value must be 'user:password':"******"requests").setLevel(logging.WARNING)
logging.getLogger("marklogic").setLevel(logging.DEBUG)
self.connection \
= Connection(args['hostname'], HTTPDigestAuth(adminuser, adminpass))
self.mls = MarkLogic(self.connection)
self.args = args
def test_list_hosts(self):
conn = Connection(tc.hostname, HTTPDigestAuth(tc.admin, tc.password))
hosts = Host.list(conn)
self.assertGreater(len(hosts), 0)
self.assertIsNotNone(hosts[0])
self.logger.info("Create simple application")
data_database = Database(self._db_name, hostname)
data_database.set_forest_names(self._forests)
modules_database = Database(self._modules_db_name, hostname)
server = HttpServer(self._http_server, "Default",
self._app_port, self._db_name, self._modules_db_name)
server.set_modules_database_name(self._modules_db_name)
data_database.create(conn)
modules_database.create(conn)
server.create(conn)
return {
u'content': data_database,
u'modules': modules_database,
u'server': server
}
if __name__ == "__main__":
logging.basicConfig(level=logging.WARNING)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("marklogic").setLevel(logging.DEBUG)
logging.getLogger("marklogic.examples").setLevel(logging.INFO)
simpleapp = SimpleApplication(tc.appname, tc.port)
conn = Connection(tc.hostname, HTTPDigestAuth(tc.admin, tc.password))
hostname = Host.list(conn)[0]
myapp = simpleapp.create(conn, hostname)
def setup_cluster(self):
if self.couple is not None and self.couple_pass is None:
self.couple_pass = self.adminpass
self.couple_user = self.adminuser
self.load_blacklist()
self.find_containers()
if not self.container_list:
print("There must be at least one unused container running.")
sys.exit(1)
if self.localimage:
ps = os.popen("docker exec " + self.container_list[0] +
" ip route")
for line in ps.readlines():
match = re.match("^default via (\S+)", line)
if match:
self.localip = match.group(1)
if self.localip is None:
print("Cannot find IP address of localhost!?")
sys.exit(1)
# Find the bootstrap image
if self.localimage:
pass
else:
self.bootimage = self.pick_image(self.bootimage)
if self.bootimage in self.container_list:
self.container_list.remove(self.bootimage)
self.cluster_list = self.pick_containers()
self.display_info()
#sys.exit(1)
# Initialize the bootstrap image, if necessary
if self.localimage:
bootip = self.localip
else:
bootip = self.ipaddr[self.bootimage]
if self.ml_init(bootip, self.bootimage):
self.ml_security(bootip, self.bootimage)
conn = Connection(bootip, HTTPDigestAuth(self.adminuser,
self.adminpass))
self.marklogic = MarkLogic(conn)
for container in self.cluster_list:
if container == self.bootimage:
continue
ip = self.ipaddr[container]
self.ml_init(ip, container)
self.ml_join(bootip, ip)
if self.name is not None:
print("{0}: rename cluster...".format(bootip))
cluster = self.marklogic.cluster()
cluster.set_cluster_name(self.name)
cluster.update()
if self.couple is not None:
for couple in self.couple:
print("{0}: couple with {1}...".format(bootip, couple))
altconn = Connection(
couple, HTTPDigestAuth(self.couple_user, self.couple_pass))
altml = MarkLogic(altconn)
altcluster = altml.cluster()
cluster = self.marklogic.cluster()
cluster.couple(altcluster)
print("Finished")
def connect(self, args):
"""Connect to the server"""
self.path = os.path.abspath(args['path'])
self.loadconfig(self.path)
if args['credentials'] is not None:
cred = args['credentials']
else:
if 'user' in self.config and 'pass' in self.config:
cred = self.config['user'] + ":" + self.config['pass']
else:
cred = None
try:
adminuser, adminpass = re.split(":", cred)
except ValueError:
raise RuntimeError("Invalid credentials (must be user:pass): {}" \
.format(args['credentials']))
if args['debug']:
logging.basicConfig(level=logging.WARNING)
logging.getLogger("requests").setLevel(logging.INFO)
logging.getLogger("marklogic").setLevel(logging.DEBUG)
self.batchsize = args['batchsize']
self.database = args['database']
self.dryrun = args['dryrun']
self.list = args['list']
self.mirror = args['mirror']
self.regex = args['regex']
self.root = args['root']
self.threshold = args['threshold']
self.verbose = args['verbose']
if self.list and self.regex:
raise RuntimeError("You must not specify both --regex and --list")
if self.root.endswith("/"):
self.root = self.root[0:len(self.root) - 1]
if args['hostname'] is None:
if 'host' in self.config:
self.hostname = self.config['host']
if 'port' in self.config:
self.port = self.config['port']
else:
self.port = 8000
if 'management-port' in self.config:
self.management_port = self.config['management-port']
else:
self.management_port = 8002
else:
parts = args['hostname'].split(":")
self.hostname = parts.pop(0)
self.management_port = 8002
self.port = 8000
if parts:
self.management_port = parts.pop(0)
if parts:
self.port = parts.pop(0)
self.connection \
= Connection(self.hostname, HTTPDigestAuth(adminuser, adminpass), \
port=self.port, management_port=self.management_port)
self.utils = ClientUtils(self.connection)
def test_no_database_found(self):
conn = Connection(tc.hostname, HTTPDigestAuth(tc.admin, tc.password))
db = Database.lookup(conn, "No-Such-Database")
self.assertIsNone(db)
def run(self, argv):
command = None
artifact = None
# This is an odd program. The parser used for each line depends on
# the command and the different parsers can be cranky about the
# allowed order of commands, options, and parameters. So we start
# by trying to "normalize" it all.
options = []
params = []
positional = []
optarg = False
for tok in argv:
if optarg:
options.append(tok)
optarg = False
elif tok.startswith("-"):
options.append(tok)
if tok != "--debug":
optarg = True
elif "=" in tok:
params.append(tok)
else:
positional.append(tok)
try:
command = positional[0]
except IndexError:
print("Usage: {0} command artifact ...".format(self.script))
sys.exit(1)
empty_artifact_commands = {
'start', 'status', 'stop', 'restart', 'init', 'save', 'switch',
'clear', 'log', 'run', 'debug'
}
try:
artifact = positional[1]
except IndexError:
if command in empty_artifact_commands:
pass
else:
print("Usage: {0} command artifact ...".format(self.script))
sys.exit(1)
# Hack for the server case
if artifact in ['http', 'xdbc', 'odbc', 'webdav']:
stype = artifact
if command == 'list':
artifact = 'servers'
else:
artifact = 'server'
positional[1] = artifact
if positional[2] == artifact:
del (positional[2])
options.append("--type")
options.append(stype)
# Hack for the stop and restart cases
if (command == 'stop' or command == 'restart') and artifact is None:
positional.append('host')
artifact = 'host'
argv = []
argv.extend(options)
argv.extend(positional)
argv.extend(params)
templ = self.cli.command_template(command)
if templ is None:
print("The command '{0}' is unrecognized".format(command))
sys.exit(1)
if artifact is None:
if 'parser' in templ:
parser = templ['parser']
else:
print("The command '{0}' isn't recognized.".format(command))
sys.exit(1)
else:
if artifact in templ:
parser = templ[artifact]["parser"]
else:
print("The command '{0}' doesn't take '{1}' artifacts.".format(
command, artifact))
sys.exit(1)
args = vars(parser.parse_args(argv))
if args['debug']:
if args['debug'] == 'debug':
# This is the debug command, not the debug option!
pass
else:
logging.basicConfig(level=logging.WARNING)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("marklogic").setLevel(logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("marklogic").setLevel(logging.INFO)
try:
username, password = re.split(":", args['credentials'])
except ValueError:
print("--credentials value must be 'user:password':"******":")[0]
try:
mgmt_port = args['hostname'].split(":")[1]
except IndexError:
mgmt_port = 8002
self.connection = Connection(host,
HTTPDigestAuth(username, password),
management_port=mgmt_port)
# do it!
if command == 'run':
self.process_script(args['script'])
else:
if artifact is None:
templ["code"](args, self.config, self.connection)
else:
templ[artifact]["code"](args, self.config, self.connection)
help="Password")
parser.add_argument("--json", action="store",
help="Name of the file containing JSON config")
parser.add_argument('--debug', action='store_true',
help='Enable debug logging')
args = parser.parse_args()
if args.debug:
logging.basicConfig(level=logging.WARNING)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("marklogic").setLevel(logging.DEBUG)
with open(args.json) as data_file:
data = json.load(data_file)
conn = Connection(args.host, HTTPDigestAuth(args.username, args.password))
# Create roles
for config in data['roles']:
name = config['role-name']
role = Role.lookup(conn, name)
if role is None:
print("Need to create role: {0}".format(name))
role = Role(name)
role.create(conn)
# Update privileges
for config in data['roles']:
name = config['role-name']
role = Role.unmarshal(config)
print("Updating role: {0}".format(name))
def backup_databases(self):
conn = Connection(self.host,
HTTPDigestAuth(self.adminuser, self.adminpass))
self.marklogic = MarkLogic(conn)
actual_databases = self.marklogic.databases()
if self.databases is None:
self.databases = actual_databases
if self.backup_root is None:
raise UnsupportedOperation("You must specify the backup root.")
if self.max_parallel <= 0:
self.max_parallel = 1
for dbname in self.databases:
if not dbname in actual_databases:
raise UnsupportedOperation("Database does not exist: {0}"
.format(dbname))
maxp = self.max_parallel
running = 0
done = False
queue = self.databases
status_list = {}
min_wait = 5
max_wait = 30
wait_incr = 5
wait = min_wait
while not done:
done = True
while len(queue) > 0 and running < maxp:
dbname = queue.pop(0)
running += 1
done = False
print("Backing up {0}".format(dbname))
if not self.dry_run:
db = self.marklogic.database(dbname)
bkp = db.backup(self.backup_root + dbname, connection=conn)
status_list[dbname] = bkp
response = bkp.status()
if response['status'] != 'in-progress':
print("{0}: {1}".format(dbname, response['status']))
if self.dry_run:
if running > 0 or len(queue) > 0:
print("{0} backups in dry-run; {1} in queue..."
.format(running, len(queue)))
running = 0
else:
if len(status_list) > 0:
new_list = {}
for dbname in status_list:
bkp = status_list[dbname]
response = bkp.status()
print("{0}: {1}".format(dbname, response['status']))
if response['status'] == 'in-progress':
done = False
new_list[dbname] = bkp
else:
running -= 1
wait = min_wait
done = done and len(queue) == 0
if not done:
status_list = new_list
if running < maxp and len(queue) != 0:
print("Running: {0} backups running; {1} in queue..."
.format(running, len(queue)))
wait = min_wait
print("")
else:
print("Waiting {0}s: {1} backups running; {2} in queue..."
.format(wait, running, len(queue)))
time.sleep(wait)
if wait < max_wait:
wait += wait_incr
print("")
