def download():
"""
_function_: `mast.datapower.web.download()`
This function is used to allow users to download ssh-transcripts.
This function will accept a form POST with two fields:
1. `hostname`: The hostname of the appliance for which the transcript
was generated. This is only used for creating the filename.
2. `content`: This is the content of the transcript and will be returned
as a download for the user.
Exposed at: `/download` via POST
Parameters:
This function accepts no arguments, but it does require a POST body
to be present in the request.
"""
# This needs to be more universal. This is currently only useful for
# ssh transcripts. This should allow any relevant file to be downloaded.
if flask.request.method == "POST":
t = Timestamp()
hostname = flask.request.form.get("hostname")
filename = '%s-%s-ssh-transcript.txt' % (t.timestamp, hostname)
f = StringIO()
f.write(
unquote(flask.request.form.get("content")).replace(
'\n', os.linesep).replace("+", " "))
f.seek(0)
return flask.send_file(f,
attachment_filename=filename,
as_attachment=True)
def status(self):
logger = make_logger("mast.datapower.status")
t = Timestamp()
check_hostname = "true" in flask.request.form.get(
'check_hostname').lower()
appliances = flask.request.form.getlist('appliances[]')
credentials = [
xordecode(_,
key=xorencode(
flask.request.cookies["9x4h/mmek/j.ahba.ckhafn"]))
for _ in flask.request.form.getlist('credentials[]')
]
if not appliances:
return flask.abort(404)
env = datapower.Environment(appliances,
credentials,
check_hostname=check_hostname)
providers = flask.request.form.getlist("providers[]")
resp = {"appliances": appliances, "time": t.short}
for provider in providers:
_provider = provider.split(".")[0]
resp[provider] = []
for appliance in env.appliances:
try:
_status = appliance.get_status(_provider)
except datapower.AuthenticationFailure:
# This is to handle an intermittent authentication failure
# sometimes issued by the DataPower. We will sleep 2
# seconds and try again
sleep(2)
try:
return self.status()
except:
logger.exception(
"An unhandled exception occurred during execution")
raise
except:
logger.exception(
"An unhandled exception occurred during execution")
raise
metric = _status.xml.find(PROVIDER_MAP[provider]).text
resp[provider].append(metric)
return flask.jsonify(resp)
def display_rows(header_row, rows, grep, provider):
_format_string = ""
for index, field in enumerate(header_row):
_ = [field] + [str(x[index]) if len(x) >= index else " " for x in rows]
length = len(max(_, key=len)) + 3
_format_string += "{%s: <%s}" % (index, length)
clear()
print Timestamp()
print provider
print "\n", _format_string.format(*header_row)
print '-' * len(_format_string.format(*header_row))
for row in rows:
for pattern in grep:
if not pattern.search(_format_string.format(*row)):
break
else:
print _format_string.format(*row)
def main(appliances=[],
credentials=[],
object_classes=[],
domains=[],
timeout=120,
no_check_hostname=False,
out_file="sample.xlsx",
delim=os.linesep,
by_appliance=False,
no_prepend_timestamp=False,
obfuscate_password=False):
prepend_timestamp = not no_prepend_timestamp
t = Timestamp()
check_hostname = not no_check_hostname
if by_appliance:
logger.info("Generating workbooks by appliance")
# we make the initial Environment to correctly handle credentials
env = Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
for appliance in env.appliances:
logger.info("generating workbook for {}".format(
appliance.hostname))
filename = os.path.split(out_file)[-1]
filename = "{}-{}".format(appliance.hostname, filename)
path = list(os.path.split(out_file)[:-1])
path.append(filename)
_out_file = os.path.join(*path)
logger.info("Workbook for {} will be stored in {}".format(
appliance.hostname, _out_file))
_env = Environment([appliance.hostname], [appliance.credentials],
timeout, check_hostname)
create_workbook(_env, object_classes, domains, _out_file, delim, t,
prepend_timestamp, obfuscate_password)
else:
logger.info("generating workbook")
env = Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
create_workbook(env, object_classes, domains, out_file, delim, t,
prepend_timestamp, obfuscate_password)
def main(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
domains=["default"],
providers=[],
delay=0.5,
out_file="./status.xlsx",
by_appliance=False,
no_prepend_timestamp=False):
prepend_timestamp = not no_prepend_timestamp
t = Timestamp()
check_hostname = not no_check_hostname
if by_appliance:
logger.info("Generating workbooks by appliance")
# we make the initial Environment to correctly handle credentials
env = Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
for appliance in env.appliances:
logger.info("generating workbook for {}".format(
appliance.hostname))
filename = os.path.split(out_file)[-1]
filename = "{}-{}".format(appliance.hostname, filename)
path = list(os.path.split(out_file)[:-1])
path.append(filename)
_out_file = os.path.join(*path)
logger.info("Workbook for {} will be stored in {}".format(
appliance.hostname, _out_file))
_env = Environment([appliance.hostname], [appliance.credentials],
timeout, check_hostname)
create_workbook(_env, domains, providers, delay, _out_file, t,
prepend_timestamp)
else:
logger.info("generating workbook")
env = Environment(appliances,
credentials,
timeout,
check_hostname=False)
create_workbook(env, domains, providers, delay, out_file, t,
prepend_timestamp)
def get_connections(env, clear_screen, class_filter, object_filter,
domain_filter, state_filter, remote_ip_filter,
remote_port_filter, local_ip_filter, local_port_filter):
if clear_screen:
clear()
else:
print "\n\n"
for appliance in env.appliances:
header = "LocalIP LocalPort RemoteIP RemotePort State Domain ServiceClass ServiceName".split(
)
print "\n", appliance.hostname, "-", Timestamp(), "\n", "=" * 133
print "{0: <16} {1: <9} {2: <16} {3: <11} {4: <15} {5: <15} {6: <30} {7: <30}".format(
*header)
print "-" * 133
matching_values = []
tcp_table = appliance.get_status("TCPTable")
for node in tcp_table.xml.findall(STATUS_XPATH):
cls = node.find("serviceClass").text or "-"
name = node.find("serviceName").text or "-"
domain = node.find("serviceDomain").text or "-"
state = node.find("state").text or "-"
remote_ip = node.find("remoteIP").text or "-"
remote_port = node.find("remotePort").text or "-"
local_ip = node.find("localIP").text or "-"
local_port = node.find("localPort").text or "-"
values = [
local_ip, local_port, remote_ip, remote_port, state, domain,
cls, name
]
if class_filter.match(cls) and object_filter.match(
name) and domain_filter.match(
domain) and state_filter.match(
state) and remote_ip_filter.match(
remote_ip) and remote_port_filter.match(
remote_port) and local_ip_filter.match(
local_ip) and local_port_filter.match(
local_port):
matching_values.append(values)
for value in matching_values:
print "{0: <16} {1: <9} {2: <16} {3: <11} {4: <15} {5: <15} {6: <30} {7: <30}".format(
*value)
def _call_method(func, kwargs):
"""Call func with kwargs if web is in kwargs, func should return a
two-tupple containing (html, request_history). Here, we write the hsitory
to a file and return the html for inclusion in the web GUI."""
import random
random.seed()
if "appliances" not in kwargs:
pass
elif not kwargs["appliances"][0]:
# Kind of a hack to return the response we want in case no appliances
# were checked in the gui
def _func(*args, **kwargs):
return ("Must select at least one appliance.",
"Must select at least one appliance.")
func = _func
if "web" in kwargs:
try:
out, hist = func(**kwargs)
except Exception, e:
# The actions implemented should handle their own exceptions,
# but if one makes it's way up here, we need to let the user know
# part of that is suppressing the exception (because otherwise
# we have no way of sending back the details)
import traceback
msg = "Sorry, an unhandled exception occurred while "
msg += "performing action:\n\n\t {}".format(str(e))
out, hist = msg, traceback.format_exc()
sys.stderr.write(traceback.format_exc())
t = Timestamp()
# TODO: move this path to configuration
filename = os.path.join("var", "www", "static", "tmp",
"request_history", t.timestamp)
if not os.path.exists(filename):
os.makedirs(filename)
rand = random.randint(10000, 99999)
_id = "{}-{}.log".format(str(t.timestamp), str(rand))
filename = os.path.join(filename, _id)
with open(filename, 'wb') as fout:
fout.write(hist)
return flask.Markup(out), _id
def upload():
"""
_function_: `mast.datapower.web.upload()`
This handles uploads of files, stores them in a temporary directory,
and returns the path and filename in JSON format
Exposed at: `/upload` via POST
Parameters:
This function does not accept any arguments, but does require a
POST body to be present in the request.
"""
global upload_dir
UPLOAD_DIRECTORY = upload_dir
t = Timestamp()
fin = flask.request.files["_file_in"]
filename = os.path.join(UPLOAD_DIRECTORY, t.timestamp)
os.makedirs(filename)
filename = os.path.join(filename, fin.filename)
fin.save(filename)
return flask.jsonify({"filename": filename})
def restore_normal_backup(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
file_in=None,
Domain="",
source_type="ZIP",
overwrite_files=True,
overwrite_objects=True,
rewrite_local_ip=True,
deployment_policy=None,
import_domain=True,
reset_domain=True,
dry_run=False,
out_dir="tmp",
web=False):
"""Restores a normal backup to the specified appliances and Domains.
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-f, --file-in`: The backup file which will be restored. This must be in the
format specified in source_type
* `-D, --Domain`: The domain to which to restore the backup
* `-s, --source-type`: The type of backup, must be either "ZIP" or "XML"
* `-N, --no-overwrite-files`: Whether to overwrite files when restoring
the backup
* `--no-overwrite-objects`: Whether to overwrite objects when restoring
the backup
* `--no-rewrite-local-ip`: Whether to rewrite the local IP Addresses
* `-d, --deployment-policy`: The deployment policy to apply when restoring
the backup
* `--no-import-domain`: Whether we are importing a domain
* `--no-reset-domain`: Whether to reset the domain
* `--dry-run`: Whether this should be a dry-run
* `-o, --out_dir`: (NOT NEEDED IN WEB GUI) The directory (local) where you would
want all of the files generated by the restore to be placed
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
logger = make_logger("mast.backups")
t = Timestamp()
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
logger.info(
"Attempting to restore normal backup on {} in {} domain".format(
str(env.appliances), Domain))
kwargs = {
"file_in": file_in,
"source_type": source_type,
"domain": Domain,
"overwrite_files": overwrite_files,
"overwrite_objects": overwrite_objects,
"rewrite_local_ip": rewrite_local_ip,
"deployment_policy": deployment_policy,
"import_domain": import_domain,
"reset_domain": reset_domain,
"dry_run": dry_run
}
resp = env.perform_action("restore_normal_backup", **kwargs)
logger.debug("Responses received {}".format(str(resp)))
out_dir = os.path.join(out_dir, "restore_normal_backup", t.timestamp)
os.makedirs(out_dir)
for host, r in resp.items():
filename = os.path.join(
out_dir, "{}-{}-{}-results.xml".format(t.timestamp, host, Domain))
with open(filename, 'wb') as fout:
fout.write(r.pretty)
if web:
return util.render_see_download_table(resp), util.render_history(env)
def get_secure_backup(appliances=[],
credentials=[],
timeout=1200,
no_check_hostname=False,
out_dir='tmp',
CryptoCertificate="",
destination='local:/raid0',
include_iscsi=False,
include_raid=False,
remove=True,
quiesce_before=True,
unquiesce_after=True,
quiesce_timeout=60,
web=False):
"""Performs a secure backup of the specified domain.
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-o, --out-dir`: (NOT NEEDED IN WEB GUI) The directory (local) to store
the backup
* `-C, --CryptoCertificate`: The CryptoCertificate object to use to encrypt
the backup
* `-d, --destination`: The base location (on the appliance) to store
the backup
* `-i, --include-iscsi`: Whether to include the iscsi filesystem
* `-I, --include-raid`: Whether to include the RAID filesystem
* `-N, --no-remove`: If specified the backup will NOT be removed from
the DataPower
* `--no-quiesce-before`: If specified, the appliance will not be
quiesced before performing the secure backup
* `--no-unquiesce-after`: If specified, the appliance will not be
unquiesced after performing the secure backup
* `-q, --quiesce-timeout`: The timeout to wait before the appliance
attempts to quiesce
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
logger = make_logger("mast.backups")
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
output = ""
if quiesce_before:
resp = {}
for appliance in env.appliances:
logger.info("Quiescing {} in preparation of Secure Backup".format(
appliance.hostname))
resp[appliance.hostname] = appliance.QuiesceDP(
timeout=quiesce_timeout)
logger.debug("Response received {}".format(
resp[appliance.hostname]))
if web:
output += util.render_boolean_results_table(
resp, suffix="Quiesce_appliance")
sleep(quiesce_timeout)
t = Timestamp()
if destination.endswith("/"):
destination = destination.rstrip("/")
destination = '%s/%s' % (destination, t.timestamp)
kwargs = {'Dir': destination, 'domain': 'default'}
logger.info("Creating directory {} on {} to store Secure Backup".format(
destination, str(env.appliances)))
resp = env.perform_async_action('CreateDir', **kwargs)
logger.debug("Responses received {}".format(str(resp)))
if web:
output += util.render_boolean_results_table(resp, suffix="CreateDir")
include_raid = 'on' if include_raid else 'off'
include_iscsi = 'on' if include_iscsi else 'off'
kwargs = {
'cert': CryptoCertificate,
'destination': destination,
'include_iscsi': include_iscsi,
'include_raid': include_raid
}
logger.info("Attempting to perform a Secure Backup on {}".format(
str(env.appliances)))
resp = env.perform_async_action('SecureBackup', **kwargs)
logger.debug("Responses received: {}".format(str(resp)))
if web:
output += util.render_boolean_results_table(resp,
suffix="SecureBackup")
if web:
results = {}
remove_results = {}
for appliance in env.appliances:
directory = os.path.join(out_dir, appliance.hostname, "SecureBackup",
t.timestamp)
start = time()
while not appliance.file_exists(
'{}/backupmanifest.xml'.format(destination), 'default'):
sleep(5)
if time() - start > timeout:
raise TimeoutError
logger.info("Attempting to retrieve Secure Backup from {}".format(
appliance.hostname))
appliance.copy_directory(destination, directory)
_directory = os.path.join(
directory,
destination.replace(":", "").replace("///", "/"))
try:
logger.info("Attempting to verify Secure Backup for {}".format(
appliance.hostname))
if appliance.verify_local_backup(_directory):
logger.info("Secure Backup integrity verified for {}".format(
appliance.hostname))
if web:
results[appliance.hostname] = "Succeeded"
else:
print '\t', appliance.hostname, " - ", "Succeeded"
if remove:
logger.info(
"Attempting to remove Secure Backup from appliance "
"{}".format(appliance.hostname))
_resp = appliance.RemoveDir(Dir=destination,
domain='default')
logger.debug("Response received: {}".format(_resp))
if web:
remove_results[appliance.hostname] = _resp
else:
logger.warn("Secure Backup for {} Corrupt!".format(
appliance.hostname))
if web:
results[appliance.hostname] = "Failed"
else:
print '\t', appliance.hostname, " - ", "Failed"
appliance.log_error('Verification of backup in %s failed' %
(_directory))
except:
if web:
results[appliance.hostname] = "Failed"
logger.exception(
"An unhandled exception occurred during execution.")
if web:
output += util.render_results_table(results,
suffix="verify-SecureBackup")
output += util.render_boolean_results_table(remove_results,
suffix="RemoveDir")
if unquiesce_after:
resp = {}
for appliance in env.appliances:
logger.info("Attempting to unquiesce {}".format(
str(appliance.hostname)))
resp[appliance.hostname] = appliance.UnquiesceDP()
logger.debug("Response received: {}".format(
resp[appliance.hostname]))
if web:
output += util.render_boolean_results_table(
resp, suffix="Unquiesce_appliance")
if web:
return output, util.render_history(env)
def get_normal_backup(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
Domain=[],
comment="",
out_dir='tmp',
individual=False,
web=False):
"""Performs a normal backup of the specified domain.
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-D, --Domain`: The domains to backup (all-domains will backup all domains)
To spcify multiple domains,
use multiple entries of the form `[-D domain1 [-D domain2...]]`
* `-C, --comment`: The comment to add to the backup
* `-o, --out-dir`: (NOT NEEDED IN WEB GUI) The directory (local) where you
would like to store the backup
* `-I, --individual`: If specified and all-domains is specified as --Domain
then backup each domain individually instead of "all-domains"
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
logger = make_logger("mast.backups")
t = Timestamp()
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
if not Domain:
raise ValueError(
"Must provide one or more domains including 'all-domains'")
if isinstance(Domain, basestring):
Domain = [Domain]
# Fixes duplicate domains issue
Domain = list(set(Domain))
results = {}
if not individual:
logger.info("Attempting to retrieve normal backup from "
"{} in {} domain".format(str(env.appliances), Domain))
kwargs = {'domains': Domain, 'comment': comment}
_results = env.perform_async_action('get_normal_backup', **kwargs)
logger.debug("backups retrieved, check file for contents")
for hostname, backup in _results.items():
directory = os.path.join(out_dir, hostname, "NormalBackup",
t.timestamp)
os.makedirs(directory)
filename = os.path.join(
directory,
'%s-%s-%s.zip' % (t.timestamp, hostname, "_".join(Domain)))
logger.debug("Writing backup for {} to {}".format(
hostname, filename))
with open(filename, 'wb') as fout:
fout.write(backup)
if _verify_zip(filename):
logger.info("backup for {} in {} domain verified".format(
hostname, str(Domain)))
results[hostname + "-" + "_".join(Domain) +
"-normalBackup"] = "Verified"
else:
logger.info("backup for {} in {} domain corrupt".format(
hostname, str(Domain)))
results[hostname + "-" + "_".join(Domain) +
"-normalBackup"] = "Corrupt"
else:
for domain in Domain:
logger.info("Attempting to retrieve normal backup from "
"{} in {} domain".format(str(env.appliances), domain))
kwargs = {'domains': domain, 'comment': comment}
_results = env.perform_async_action('get_normal_backup', **kwargs)
logger.debug("backups retrieved, check file for contents")
for hostname, backup in _results.items():
directory = os.path.join(out_dir, hostname, "NormalBackup",
t.timestamp)
if not os.path.exists(directory):
os.makedirs(directory)
filename = os.path.join(
directory,
'%s-%s-%s.zip' % (t.timestamp, hostname, domain))
logger.debug("Writing backup for {} to {}".format(
hostname, filename))
with open(filename, 'wb') as fout:
fout.write(backup)
if _verify_zip(filename):
logger.info("backup for {} in {} domain verified".format(
hostname, domain))
results[hostname + "-" + domain +
"-normalBackup"] = "Verified"
else:
logger.info("backup for {} in {} domain corrupt".format(
hostname, domain))
results[hostname + "-" + domain +
"-normalBackup"] = "Corrupt"
if web:
return util.render_results_table(results), util.render_history(env)
for k, v in results.items():
print
print k
print '=' * len(k)
print v
print
import os
import openpyxl
from time import sleep
from mast.cli import Cli
from mast.logging import make_logger
from mast.datapower import datapower
from mast.timestamp import Timestamp
from mast.datapower.datapower import Environment
t = Timestamp()
logger = make_logger("mast.getstatus", propagate=False)
def recurse_status(prefix, elem, row, header_row):
for child in elem.findall(".//*"):
logger.info("Found child {}".format(child.tag))
if child.tag not in header_row:
logger.debug("Appending {} to header row".format(child.tag))
header_row.append(child.tag)
if len(child):
_prefix = "{}/{}".format(prefix, child.tag)
logger.info("child {} has children, recursing...".format(_prefix))
recurse_status(_prefix, child, row, header_row)
else:
logger.info("Adding {} -> {}".format(child.tag, child.text))
row.insert(header_row.index(child.tag), child.text)
def create_workbook(env, domains, providers, delay, out_file, timestamp,
prepend_timestamp):
def make_logger(
name,
level=level,
fmt=_format,
max_bytes=max_bytes,
backup_count=backup_count,
delay=delay,
propagate=propagate,
):
"""
_function_: `mast.logging.make_logger(name, level=level, fmt=_format, filename=None, when=unit, interval=interval, propagate=propagate, backup_count=backup_count)`
Returns an instance of logging.Logger configured with
a [logging.handlers.TimedRotatingFileHandler](https://docs.python.org/2/library/logging.handlers.html#timedrotatingfilehandler)
handler.
Arguments passed to this function determine the format,
level, filename, time unit, interval, backup count and
whether to propagate messages to parent loggers (defined
by dot seperated heirarchy ie in `mast.datapower`,
`datapower` is a logger with a parent logger of mast).
Parameters:
* `name`: Required. the name of the logger instance. This follows
conventions mentioned [here](https://docs.python.org/2/library/logging.html#logger-objects)
* `level`: The logging level to listen for. Accepts an `int` or
one of the logging modules convenience constants defined
[here](https://docs.python.org/2/library/logging.html#logging-levels)
* `fmt`: The format of the log message, see
[here](https://docs.python.org/2/library/logging.html#formatter-objects)
and [here](https://docs.python.org/2/library/logging.html#logrecord-attributes)
for more details
* `filename`: The filename to log to. Defaults to the name of the logger
appended with `.log` in the `$MAST_HOME/var/log` directory or
`$MAST_HOME/var/log/mastd` directory if running as `mastd`
* `when`: The time unit to use for rolling over the log file
as detailed [here](https://docs.python.org/2/library/logging.handlers.html#timedrotatingfilehandler)
* `interval`: The number of time units to wait before rolling the
log files as detailed [here](https://docs.python.org/2/library/logging.handlers.html#timedrotatingfilehandler)
* `propagate`: Whether to propagate log messages up the ancestry chain
(ie. if you have a logger `mast.datapower`, and propagate is set to
`False` messages sent to the this logger will not be propagated to the
`mast` logger). See [here](https://docs.python.org/2/library/logging.html#logging.Logger.propagate)
for more details.
* `backup_count`: The number of "rolled" log files to keep, see
[here](https://docs.python.org/2/library/logging.handlers.html#timedrotatingfilehandler)
for more details.
Usage:
:::python
from mast.logging import make_logger
logger = make_logger("my_module")
logger.info("informational message")
logger.debug("debug message")
"""
t = Timestamp()
_logger = logging.getLogger(name)
if _logger.handlers:
if len(_logger.handlers) >= 1:
return _logger
_logger.setLevel(level)
_formatter = logging.Formatter(fmt)
pid = os.getpid()
directory = os.path.join(
mast_home,
"var",
"log",
getpass.getuser(),
"{}-{}".format(t.timestamp, str(pid)),
)
filename = os.path.join(
directory,
"{}-{}.log".format(name, t.timestamp),
)
if not os.path.exists(directory):
os.makedirs(directory)
_handler = RotatingFileHandler(
filename,
maxBytes=max_bytes,
backupCount=backup_count,
delay=delay,
)
_handler.setFormatter(_formatter)
_handler.setLevel(level)
_logger.addHandler(_handler)
_logger.propagate = propagate
return _logger
def clean_up(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
Domain='default',
checkpoints=False,
export=False,
error_reports=False,
recursive=False,
logtemp=False,
logstore=False,
backup_files=True,
out_dir='tmp',
web=False):
"""This will clean up the specified appliances filesystem optionally
(defaults to True) taking copies of the files as backups.
Parameters:
* `-a, --appliances` - The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-D, --Domain`: The domain who's filesystem you would like to clean up
* `-C, --checkpoints`: If specified, all checkpoints will be removed
from the domain
* `-e, --export`: If specified all exports will be removed from the domain
* `-l, --logtemp`: If specified, all files in `logtemp:` will be removed
from the domain
* `-L, --logstore`: If specified, all files in `logstore:` will be
removed
* `-E, --error-reports`: If specified, all error reports will be removed
from the appliance(s)
* `-r, --recursive`: If specified, directories will be cleaned recursively
* `--no-backup-files`: If specified, files will not be backed up before
deleting
* `-o, --out-dir`: The directory to save backed up files
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
t = Timestamp()
dirs = []
if checkpoints:
dirs.append('chkpoints:/')
if export:
dirs.append('export:/')
if logtemp:
dirs.append('logtemp:/')
if logstore:
dirs.append('logstore:/')
if web:
rows = []
for appliance in env.appliances:
if web:
rows.append((appliance.hostname, ))
for _dir in dirs:
_clean_dir(appliance, _dir, Domain, recursive, backup_files,
t.timestamp, out_dir)
if web:
rows.append(("", _dir, "Cleaned"))
if error_reports:
_clean_error_reports(appliance, Domain, backup_files, t.timestamp,
out_dir)
rows.append(("", "ErrorReports", "Cleaned"))
return flask.render_template(
"results_table.html",
header_row=["Appliance", "Location", "Action"],
rows=rows), util.render_history(env)
def call_method(plugin, form):
"""Gather the arguments and function name from form then invoke
_call_method. Wrap the results in html and return them."""
t = Timestamp()
name = form.get("callable")
arguments, func = _get_arguments(plugin, name)
kwargs = {}
for arg, default in arguments:
if isinstance(default, bool):
if arg == "web":
kwargs[arg] = True
continue
value = form.get(arg)
if value == 'true':
kwargs[arg] = True
else:
kwargs[arg] = False
elif isinstance(default, list):
# TODO: This needs to implement a selection feature
if arg == 'appliances':
kwargs[arg] = form.getlist(arg + '[]')
elif arg == 'credentials':
kwargs[arg] = [
xordecode(
_,
key=xorencode(
flask.request.cookies["9x4h/mmek/j.ahba.ckhafn"]))
for _ in form.getlist(arg + '[]')
]
else:
kwargs[arg] = form.getlist(arg + '[]')
elif isinstance(default, basestring):
if arg == 'out_dir':
kwargs[arg] = os.path.join('tmp', 'web', name, t.timestamp)
elif arg == 'out_file' and default is not None:
kwargs[arg] = os.path.join(
"tmp", "web", name,
"{}-{}{}".format(t.timestamp, name,
os.path.splitext(default)[1])).replace(
os.path.sep, "/")
else:
kwargs[arg] = form.get(arg) or default
elif isinstance(default, int):
kwargs[arg] = int(form.get(arg)) or default
elif default is None:
kwargs[arg] = form.get(arg) or default
out, history_id = _call_method(func, kwargs)
link = ""
if 'out_dir' in kwargs:
config = get_config("server.conf")
static_dir = config.get('dirs', 'static')
fname = ""
for appliance in kwargs['appliances']:
fname = "{}-{}".format(fname, appliance)
fname = "{}-{}{}.zip".format(t.timestamp, name, fname)
zip_filename = os.path.join(static_dir, 'tmp', fname)
zip_file = zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED)
_zipdir(kwargs['out_dir'], zip_file)
zip_file.close()
#filename = '%s-%s.zip' % (t.timestamp, name)
link = flask.Markup(flask.render_template('link.html', filename=fname))
if 'out_file' in kwargs and kwargs["out_file"] is not None:
import shutil
config = get_config("server.conf")
static_dir = config.get('dirs', 'static')
dst = os.path.join(static_dir, "tmp",
os.path.basename(kwargs["out_file"]))
shutil.copyfile(kwargs["out_file"], dst)
link = flask.Markup(
flask.render_template('link.html',
filename=os.path.basename(
kwargs["out_file"])))
out = flask.render_template('output.html',
output=out,
callable=name,
timestamp=str(t),
history_id=history_id,
link=link)
if 'out_dir' in kwargs:
out = out
return out
dst_dir = os.path.join(dst, directory)
if not os.path.exists(dst_dir):
os.mkdir(dst_dir)
for f in files:
print os.path.join(root, f), "->", dst
shutil.copy(os.path.join(root, f), dst)
print "\n\nhotfix installed"
print "\nTo re-generate the latest documentation, please run"
print "\t{}mast contrib/gendocs.py".format(mast_home + os.path.sep)
if __name__ == "__main__":
main()
'''
now = Timestamp().timestamp
mast_home = os.environ["MAST_HOME"]
default_out_file = os.path.join(
mast_home,
"tmp",
"hotfix.{}.zip".format(now))
default_build_dir = os.path.join(
mast_home,
"tmp",
"{}-hotfix-build".format(now))
repos = [
"https://github.com/McIndi/mast.cli/archive/master.zip",
"https://github.com/McIndi/mast.config/archive/master.zip",
def main(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
domains=["default"],
providers=[],
object_classes=[],
delim=os.linesep,
delay=0.5,
out_file="./status.xlsx",
by_appliance=False,
no_prepend_timestamp=False,
obfuscate_password=False,
persisted=False):
r"""
_Script_: `contrib/dp-query.py`
DESCRIPTION:
This script will create an excel workbook generated from the output
from the specified appliances for arbitrary status providers and/or
object configuration. Each status provider and object class configuration
will be contained in a seperate worksheet
RETURNS:
An Excel workbook
USAGE:
```bash
$ mast contrib/dp-query --help
$ mast contrib/dp-query --appliances APPLIANCE --credentials CREDS --domains all-domains --providers ServicesStatusPlus --object-classes MultiProtocolGateway HTTPFrontSideHandler
```
PARAMETERS:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-d, --domains`: The domains to query. To pass multiple arguments
to this parameter, use multiple entries of the form
`[-d domain1 [-d domain2...]]`. If `all-domains` is passed to this
parameter, all domains will be queried.
* `-p, --providers`: The status providers to query. To pass multiple
arguments to this parameter, use multiple entries of the form
`[-p provider1 [-p provider2...]]`.
* `-o, --object-classes`: The object classes to pull the configuration
of. To pass multiple arguments to this parameter, use multiple entries
of the form `[-o objectclass1 [-o objectclass2...]]`.
* `-D, --delim`: The delimeter to use when multiple values will end up in
the same cell. Defaults to your system's default line termination
character (`\r\n` on windows or `\n` on Linux)
* `--delay`: The amount of time (in seconds) to wait between requests
to the appliances. This can be extremely useful if you need to avoid
over-working the appliance for example if you need to gather information
during peak periods you would probably want to set this to one or more
seconds. Defaults to `0.5`.
* `-O, --out-file`: The path and filename of the output file. This will
be an excel workbook, so it should have the extention `.xlsx`
* `-b, --by-appliance`: If specified, a seperate excel workbook will
be generated for each appliance.
* `-N, --no-prepend-timestamp`: If specified, this script will not
prepend a timestamp to the output filename
* `--obfuscate-password`: If specified, any object configurations which
contain a password will have its value replaced with "********",
that's eight asterics.
* `-P, --persisted`: If specified, the persisted configuration of any
specified object classes will be retrieved instead of the running
configuration
"""
prepend_timestamp = not no_prepend_timestamp
t = Timestamp()
check_hostname = not no_check_hostname
if by_appliance:
logger.info("Generating workbooks by appliance")
# we make the initial Environment to correctly handle credentials
env = Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
for appliance in env.appliances:
logger.info("generating workbook for {}".format(
appliance.hostname))
filename = os.path.split(out_file)[-1]
filename = "{}-{}".format(appliance.hostname, filename)
path = list(os.path.split(out_file)[:-1])
path.append(filename)
_out_file = os.path.join(*path)
logger.info("Workbook for {} will be stored in {}".format(
appliance.hostname, _out_file))
_env = Environment([appliance.hostname], [appliance.credentials],
timeout, check_hostname)
create_workbook(_env, domains, providers, object_classes, delay,
delim, _out_file, t, prepend_timestamp,
obfuscate_password, persisted)
else:
logger.info("generating workbook")
env = Environment(appliances,
credentials,
timeout,
check_hostname=False)
create_workbook(env, domains, providers, object_classes, delay, delim,
out_file, t, prepend_timestamp, obfuscate_password,
persisted)
def postdeploy(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
Domain="",
unquiesce_domain=True,
unquiesce_appliance=False,
postdeploy_command=None,
save_config=True,
web=False):
"""This is a simple script which will allow you to unquiesce
your domain or appliances after you quiesce them for a deployment.
Also this will allow you to save the config.
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-D, --Domain`: The domain which will be unquiesced and persisted.
* `-N, --no-unquiesce-domain`: If specified, this script will not attempt
to unquiesce the domain
* `-u, --unquiesce-appliance`: If specified, this script will attempt to
unquiesce the appliance
* `-p, --postdeploy-command`: This command will be "shelled out"
to the machine running MAST after unquiescing and saving the configuration,
use this parameter to clean up VCS artifacts and/or perform check-outs
of your deployed services or similar operations
* `--no-save-config`: If specified, the configuration will not be saved in
the application domain
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
import mast.datapower.system as system
check_hostname = not no_check_hostname
logger = make_logger('mast.datapower.deployment')
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
if web:
output = ""
history = ""
for appliance in env.appliances:
if unquiesce_appliance:
appliance.log_info("Attempting to unquiesce appliance")
_out = system.unquiesce_appliance(
appliances=[appliance.hostname],
credentials=[appliance.credentials],
timeout=timeout,
no_check_hostname=no_check_hostname,
web=web)
appliance.log_info("Finished Unquiescing appliance")
if web:
output += _out[0]
history += _out[1]
else:
print "Finished unquiescing appliance"
if unquiesce_domain:
appliance.log_info("Attempting to unquiesce domain")
_out = system.unquiesce_domain(appliances=[appliance.hostname],
credentials=[appliance.credentials],
timeout=timeout,
no_check_hostname=no_check_hostname,
Domain=Domain,
web=web)
appliance.log_info("Finished Unquiescing domain")
if web:
output += _out[0]
history += _out[1]
else:
print "Finished unquiescing domain"
if save_config:
appliance.log_info(
"Attempting to save configuration after deployment")
_out = system.save_config(appliances=[appliance.hostname],
credentials=[appliance.credentials],
timeout=timeout,
no_check_hostname=no_check_hostname,
Domain=Domain,
web=web)
appliance.log_info(
"Finished saving configuration after deployment")
if web:
output += _out[0]
history += _out[1]
else:
print "Finished saving the configuration"
if postdeploy_command:
logger.info(
"Post-Deployment command '{}' found. Executing at {}".format(
postdeploy_command, str(Timestamp())))
out, err = system_call(command=postdeploy_command)
out = str(out)
err = str(err)
logger.info(
"finished executing Post-Deployment command '{}' at {}., output: {}"
.format(postdeploy_command, str(Timestamp()), ";".join([out,
err])))
if web:
from mast.plugin_utils.plugin_utils import render_results_table
results = {
"postdeploy command":
"{}\n\nout: {}\n\nerr: {}".format(postdeploy_command, out, err)
}
output += render_results_table(results)
else:
print "Finished running post-deploy command. output: {}".format(
";".join([out, err]))
if web:
return output, history
def deploy(appliances=[],
credentials=[],
timeout=180,
no_check_hostname=False,
Domain=[],
file_in=None,
deployment_policy="",
dry_run=False,
overwrite_files=True,
overwrite_objects=True,
rewrite_local_ip=True,
object_audit=True,
out_dir='tmp',
format='ZIP',
quiesce_domain=True,
quiesce_appliance=False,
quiesce_timeout=120,
web=False):
"""Perform a deployment/migration of a service/object to an IBM DataPower
appliance. This script will try to perform the deployment/migration in a
manner consistent with best practices.
__WARNING__: There is an inherent security risk involved in this script,
in order to allow the most flexible integration with various
Version Control Systems possible, we allow a pre-deployment
hook and a post-deployment hook which will be "shelled out"
to your operating system. For this reason PLEASE be sure to
run this script (and the MAST Web GUI server) as a user with
appropriate permissions.
DO NOT RUN AS ROOT!!!
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-D, --Domain`: The domain to import the configuration into, also the
domain to quiesce if `--no-quiesce-domain` is not specified.
* `-f, --file-in`: This is the configuration file that you are
importing. It must be in the format specified by the format
parameter.
* `-d, --deployment-policy`: The deployment policy to apply to the
import, must already exist on the appliance
* `--dry-run`: If specified, a dry-run will be performed instead of
an actual import
* `-N, --no-overwrite-files`: If specified, no files will be over-
written during the import
* `--no-overwrite-objects`: If specified, no objects will be over-
written during import
* `--no-rewrite-local-ip`: If specified local ip addresses will not
be rewritten on import
* `--no-object-audit`: If specified, an object audit will not be
performed. An object audit is a diff between the running and
persisted configuration
* `-o, --out-dir`: This is where to place the artifacts generated
by this script
* `-F, --format`: The format of the configuration file, must be
either "ZIP" or "XML".
* `--no-quiesce-domain`: If specified, the domain will not be quiesced
prior to the deployment.
* `-q, --quiesce-appliance`: If specified, the appliance will be
quiesced prior to the deployment
* `-Q, --quiesce-timeout`: This is the amount of time for the appliance
to wait before beginning the quiescence procedure.
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
import mast.datapower.system as system
from mast.datapower.developer import _import
logger = make_logger('mast.datapower.deployment')
if web:
output = ""
history = ""
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
for appliance in env.appliances:
appliance.log_info("Deployment started on {}".format(
appliance.hostname))
# Quiesce Domain
if quiesce_domain:
logger.info("Quiescing domain {} before deployment at {}".format(
Domain, str(Timestamp())))
_out = system.quiesce_domain(appliances=[appliance.hostname],
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
Domain=Domain,
quiesce_timeout=quiesce_timeout,
web=web)
logger.info(
"Finished quiescing domain {} before deployment at {}".format(
Domain, str(Timestamp())))
sleep(quiesce_timeout)
if web:
output += _out[0]
history += _out[1]
# Quiesce Appliance
if quiesce_appliance:
logger.info("Quiescing appliances before deployment at {}".format(
str(Timestamp())))
_out = system.quiesce_appliance(
appliances=appliance.hostname,
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
quiesce_timeout=quiesce_timeout,
web=web)
logger.info(
"Finished quiescing appliances before deployment at {}".format(
str(Timestamp())))
sleep(quiesce_timeout)
if web:
output += _out[0]
history += _out[1]
appliance.log_info("Attempting to import configuration at '{}'".format(
str(Timestamp())))
file_out = os.path.join(
out_dir, '{}-deployment_results.txt'.format(appliance.hostname))
# import configuration
_out = _import(appliances=[appliance.hostname],
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
Domain=Domain,
file_in=file_in,
deployment_policy=deployment_policy,
dry_run=dry_run,
overwrite_files=overwrite_files,
overwrite_objects=overwrite_objects,
rewrite_local_ip=rewrite_local_ip,
source_type=format,
out_dir=out_dir,
web=web)
if web:
output += _out[0]
history += _out[1]
appliance.log_info("Finished importing configuration at {}".format(
str(Timestamp())))
# unquiesce domain
if quiesce_domain:
appliance.log_info("Attempting to unquiesce domain")
_out = system.unquiesce_domain(appliances=[appliance.hostname],
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
Domain=Domain,
web=web)
appliance.log_info("Finished unquiescing domain")
if web:
output += _out[0]
history += _out[1]
# unquiesce appliance
if quiesce_appliance:
logger.info("Quiescing appliances before deployment at {}".format(
str(Timestamp())))
_out = system.unquiesce_appliance(
appliances=[appliance.hostname],
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
web=web)
logger.info(
"Finished quiescing appliances before deployment at {}".format(
str(Timestamp())))
if web:
output += _out[0]
history += _out[1]
if object_audit:
appliance.log_info(
"Post-Deployment Object audit started at {}".format(
str(Timestamp())))
_out = system.objects_audit(appliances=[appliance.hostname],
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
out_dir=out_dir,
web=web)
appliance.log_info(
"Post-Deployment Object audit finished at {}".format(
str(Timestamp())))
if web:
output += _out[0]
history += _out[1]
if web:
return output, history
def predeploy(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
out_dir="tmp",
Domain="",
comment="",
predeploy_command=None,
CryptoCertificate="",
secure_backup_destination="local:/raid0",
backup_default=True,
backup_all=True,
do_secure_backup=False,
do_normal_backup=True,
set_checkpoints=True,
include_iscsi=False,
include_raid=False,
remove_secure_backup=True,
default_checkpoint=True,
remove_oldest_checkpoint=True,
allow_shell=False,
web=False):
"""Perform routine pre-deployment actions. Everything is optional, but if
you wish to perform an action, you must provide the necessary arguments.
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-o, --out-dir`: (Not needed in web GUI) The directory (local)
where you would like to store the artifacts generated by this
script
* `-D, --Domain`: This is the app domain to backup and the app domain
in which to set a checkpoint
* `-C, --comment`: This is shared among other actions. The comment is
used to build the name of the checkpoint.
* `-p, --predeploy-command`: This command will be "shelled out"
to the machine running MAST after performing the backups and checkpoints.
Use this parameter to pull from version control or similar
operations.
* `--CryptoCertificate`: The CryptoCertificate with which to
encrypt the secure backup
* `-s, --secure-backup-destination`: The destination (on the DataPower)
where the secure backup will be stored
* `-N, --no-backup-default`: Whether to also backup the default domain
* `--no-backup-all`: Whether to also backup all-domains
* `-d, --do-secure-backup`: Whether to retrieve a secure backup
* `--no-do-normal-backup`: Whether to retrieve normal backups
* `--no-set-checkpoints`: Whether to set checkpoints
* `-i, --include-iscsi`: Whether to include the iscsi volume in the
secure backup
* `-I, --include-raid`: Whether to include the raid volume in the
secure backup
* `--no-remove-secure-backup`: Whether to remove the secure backup
from the appliance after verifying your local copy.
* `--no-default-checkpoint`: Whether to create a checkpoint in the
default domain
* `--no-remove-oldest-checkpoint`: Whether to remove the oldest
checkpoint from the domain IF AND ONLY IF the maximum number
of checkpoints has been reached.
* `-A, --allow-shell`: Whether to allocate a shell for the execution of
the predeploy-command
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__
Here is what is possible (will be done in this order):
1. Secure Backup
* The following parameters apply:
* `-d, --do-secure-backup`
* `-o, --out-dir`
* `--CryptoCertificate`
* `-s, --secure-backup-destination`
* `-i, --include-iscsi`
* `-I, --include-raid`
* `--no-remove-secure-backup`
2. Normal Backups
* The following parameters apply:
* `--no-do-normal-backup`
* `-o, --out-dir`
* `-D, --Domain`
* `-N, --no-backup-default`
* `--no-backup-all`
* `-C, --comment`
3. Checkpoints
* The following parameters apply:
* `--no-set-checkpoints`
* `-D, --Domain`
* `-C, --comment`
* `--no-default-checkpoint`
* `--no-remove-oldest-checkpoint`"""
from mast.datapower.backups import set_checkpoint
from mast.datapower.backups import get_normal_backup, get_secure_backup
logger = make_logger('mast.datapower.deployment')
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
if web:
output = ""
history = ""
# Loop through appliances so we will only affect one appliance at a time
for appliance in env.appliances:
# Secure Backup
if do_secure_backup:
if not CryptoCertificate:
# Fail if CryptoCertificate is not given
logger.error("Cert must be specified in order "
"to perform a secure backup!")
sys.exit(-1)
logger.info("Starting Secure Backup for {}".format(
appliance.hostname))
_out = get_secure_backup(appliances=appliance.hostname,
credentials=appliance.credentials,
timeout=timeout,
out_dir=out_dir,
CryptoCertificate=CryptoCertificate,
destination=secure_backup_destination,
include_iscsi=include_iscsi,
include_raid=include_raid,
remove=remove_secure_backup,
quiesce_before=False,
unquiesce_after=False,
no_check_hostname=no_check_hostname,
web=web)
if web:
output += _out[0]
history += _out[1]
logger.info("Finished Secure Backup for {}".format(
appliance.hostname))
# Normal backups
if do_normal_backup:
logger.info("Pre-Deployment backups started at {}".format(
str(Timestamp())))
domains = [Domain]
if backup_default:
domains.append("default")
if backup_all:
domains.append("all-domains")
_out = get_normal_backup(appliances=[appliance.hostname],
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
Domain=domains,
comment=comment,
out_dir=out_dir,
web=web)
logger.info("Pre-Deployment backups finished at {}".format(
str(Timestamp())))
if web:
output += _out[0]
history += _out[1]
# Checkpoints
if set_checkpoints:
logger.info("Pre-Deployment checkpoints started at {}".format(
str(Timestamp())))
domains = [Domain]
if default_checkpoint:
domains.append("default")
_out = set_checkpoint(appliances=[appliance.hostname],
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
Domain=domains,
comment=comment,
remove_oldest=remove_oldest_checkpoint,
web=web)
logger.info("Pre-Deployment checkpoints finished at {}".format(
str(Timestamp())))
if web:
output += _out[0]
history += _out[1]
if predeploy_command:
logger.info(
"Pre-Deployment command '{}' found. Executing at {}".format(
predeploy_command, str(Timestamp())))
out, err = system_call(command=predeploy_command, shell=allow_shell)
out = str(out)
err = str(err)
logger.info(
"finished executing Pre-Deployment command '{}' at {}., output: {}"
.format(predeploy_command, str(Timestamp()), ";".join([out, err])))
if web:
from mast.plugin_utils.plugin_utils import render_results_table
results = {
"predeploy command":
"{}\n\nout: {}\n\nerr: {}".format(predeploy_command, out, err)
}
output += render_results_table(results)
else:
print "Finished running pre-deploy command. output: {}".format(
";".join([out, err]))
if web:
return output, history
def set_checkpoint(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
Domain=['default'],
comment='',
remove_oldest=True,
web=False):
"""Sets a checkpoint in the given domains on the specified appliances
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-D, --Domain`: Domains to set checkpoints in. To spcify multiple domains,
use multiple entries of the form `[-D domain1 [-D domain2...]]`
* `-C, --comment`: The comment to use for the checkpoint (will also be used to
build the checkpoint name)
* `-N, --no-remove-oldest`: If specified this script will attempt to
remove the oldest checkpoint __IF__ the maximum number of checkpoints exist
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
logger = make_logger("mast.backups")
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
logger.info("Attempting to set checkpoint on {} in {} domain(s)".format(
str(env.appliances), str(Domain)))
t = Timestamp()
if web:
header_row = ("Appliance", "Result")
rows = []
for appliance in env.appliances:
if not web:
print appliance.hostname
_domains = Domain
print Domain
print _domains
if "all-domains" in _domains:
_domains = appliance.domains
print _domains
for domain in _domains:
print domain
if not web:
print "\t", domain
name = '{0}-{1}-{2}'.format(comment, domain, t.timestamp)
logger.debug(
"Attempting to set checkpoint {} on {} in {} domain".format(
name, appliance, domain))
if remove_oldest:
_max = appliance.max_checkpoints(domain)
if len(appliance.get_existing_checkpoints(domain)) >= _max:
logger.info(
"Maximum number of checkpoints for domain "
"{} on {} reached. Removing oldest checkpoint.".format(
domain, appliance.hostname))
_resp = appliance.remove_oldest_checkpoint(domain)
logger.debug("Response received: {}".format(_resp))
kwargs = {'domain': domain, 'ChkName': name}
resp = appliance.SaveCheckpoint(**kwargs)
logger.debug("Response received: {}".format(resp))
if not web:
if resp:
print "\t\tSuccessful"
else:
print "\t\tFailed"
if web:
if resp:
rows.append(
("{}-{}-set_checkpoint".format(appliance.hostname,
domain), "Succeeded"))
else:
rows.append(
("{}-{}-set_checkpoint".format(appliance.hostname,
domain), "Failed"))
if web:
return flask.render_template("results_table.html",
header_row=header_row,
rows=rows), util.render_history(env)
def _import(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
Domain=[],
file_in=None,
deployment_policy=None,
deployment_policy_variables=None,
dry_run=False,
overwrite_files=True,
overwrite_objects=True,
rewrite_local_ip=True,
source_type='ZIP',
out_dir="tmp/",
web=False):
"""Import a service/object into the specified appliances
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-D. --Domain`: The domain into which the configuration will be imported
* `-f, --file-in`: The file to import into the specified domain. This
__MUST__ match the format specified in source_type
* `-d, --deployment-policy`: The deployment policy to use for the import
(must already exist on the appliances)
* `--dry-run`: Whether to do a dry-run (nothing will be imported)
* `-N, --no-overwrite-files`: If specified, no files will be overwritten
as part of the import
* `--no-overwrite-objects`: If specified, no objects will be overwritten
as part of the import
* `--no-rewrite-local-ip`: If specified, no local ip addresses will be
rewritten as part of the import
* `-s, --source-type`: The type of file to import. Can be "XML" or "ZIP"
* `-o, --out-dir`: The directory to output artifacts generated by this
script
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
logger = make_logger("mast.developer")
t = Timestamp()
check_hostname = not no_check_hostname
env = datapower.Environment(
appliances,
credentials,
timeout,
check_hostname=check_hostname)
msg = "Attempting to import {} to {}".format(file_in, str(env.appliances))
logger.info(msg)
if not web:
print msg
results = {}
out_dir = os.path.join(out_dir, "import_results", t.timestamp)
os.makedirs(out_dir)
for appliance in env.appliances:
if not web:
print appliance.hostname
results[appliance.hostname] = {}
domains = Domain
if "all-domains" in domains:
domains = appliance.domains
for domain in domains:
if not web:
print "\t", domain
kwargs = {
'domain': domain,
'zip_file': file_in,
'deployment_policy': deployment_policy,
'deployment_policy_variables': deployment_policy_variables,
'dry_run': dry_run,
'overwrite_files': overwrite_files,
'overwrite_objects': overwrite_objects,
'rewrite_local_ip': rewrite_local_ip,
'source_type': source_type}
resp = appliance.do_import(**kwargs)
results[appliance.hostname][domain] = resp
if not web:
pprint_xml(resp.xml)
logger.debug("Response received: {}".format(str(resp)))
filename = os.path.join(
out_dir,
"{}-{}-import_results.xml".format(
appliance.hostname,
domain
)
)
with open(filename, 'wb') as fout:
fout.write(resp.pretty)
if web:
return util.render_see_download_table(
results, suffix="import"), util.render_history(env)
def export(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
Domain="",
object_name=None,
object_class=None,
comment='',
format='ZIP',
persisted=True,
all_files=True,
referenced_files=True,
referenced_objects=True,
out_dir='tmp',
web=False):
"""Exports a service or object to be used to import into another
domain or appliance
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-D, --Domain`: The domain from which to export service/object
* `-o, --object-name`: The name of the object to export
* `-O, --object-class`: The class of the object to export
* `-C, --comment`: The comment to embed into the export
* `-f, --format`: the format in which to export the configuration. This
can be either "XML" or "ZIP"
* `-N, --no-persisted`: If specified, the running configuration will be
exported as opposed to the persisted configuration
* `--no-all-files`: If specified, the export will not include all files
* `--no-referenced-files`: If specified, the referenced files will not
be included in the export
* `--no-referenced-objects`: If specified, referenced objects will not
be included in the export.
* `--out-dir`: (**NOT NEEDED IN THE WEB GUI**)The directory (local)
in which to save the export
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
logger = make_logger("mast.developer")
t = Timestamp()
if object_name is None or object_class is None:
try:
raise TypeError("Must Provide both object name and object class")
except:
logger.exception("Must Provide both object name and object class")
raise
check_hostname = not no_check_hostname
env = datapower.Environment(
appliances,
credentials,
timeout,
check_hostname=check_hostname)
msg = "Attempting to export {} from {}".format(object_name, str(env.appliances))
logger.info(msg)
kwargs = {
'domain': Domain,
'obj': object_name,
'object_class': object_class,
'comment': comment,
'format': format,
'persisted': persisted,
'all_files': all_files,
'referenced_objects': referenced_objects,
'referenced_files': referenced_files}
results = env.perform_action(
'export',
**kwargs)
for hostname, _export in results.items():
d = os.path.join(out_dir, hostname, t.timestamp)
os.makedirs(d)
extention = format.lower()
filename = os.path.join(d, '%s-%s-%s.%s' % (
t.timestamp,
hostname,
object_name,
extention))
msg = "Writing export of {} from {} to {}".format(object_name, hostname, filename)
logger.debug(msg)
if not web:
print msg
with open(filename, 'wb') as fout:
fout.write(_export)
if web:
return util.render_see_download_table(
results, suffix="export"), util.render_history(env)
from mast.timestamp import Timestamp
from mast.datapower import datapower
from mast.logging import make_logger
from cStringIO import StringIO
from pygments import highlight
import mast.pprint as pprint
from mast.cli import Cli
import colorama
import urllib2
import os
import re
colorama.init()
default_base_dir = os.path.join(os.environ["MAST_HOME"], "tmp", "config")
default_comment = "Update {}".format(Timestamp().friendly)
def main(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
base_dir=default_base_dir,
comment=default_comment,
persisted=False,
recursive=False,
no_strip_timestamp=False,
page=False):
"""
track_getconfig.py
def get_file(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
location=None,
Domain='default',
out_dir='tmp',
web=False):
"""Retrieves a file from the specified appliances
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-l, --location`: The location of the file (on DataPower) you would
like to get
* `-D, --Domain`: The domain from which to get the file
* `-o, --out-dir`: (NOT NEEDED IN THE WEB GUI)The directory you would like to
save the file to
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
t = Timestamp()
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
kwargs = {'domain': Domain, 'filename': location}
responses = env.perform_async_action('getfile', **kwargs)
if not os.path.exists(out_dir) or not os.path.isdir(out_dir):
os.makedirs(out_dir)
for hostname, fin in list(responses.items()):
filename = location.split('/')[-1]
filename = os.path.join(out_dir,
'%s-%s-%s' % (hostname, t.timestamp, filename))
with open(filename, 'wb') as fout:
fout.write(fin)
if web:
return util.render_see_download_table(
responses, suffix="get_file"), util.render_history(env)
