def set_file(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
file_in=None,
destination=None,
Domain='default',
overwrite=True,
web=False):
"""Uploads a file to the specified appliances
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-f, --file-in`: The path and filename of the file to upload
* `-d, --destination`: Should be the path and filename of the file
once uploaded to the DataPower **NOTE: file_out should contain
the filename ie. local:/test.txt**
* `-D, --Domain`: The domain to which to upload the file,
* `-N, --no-overwrite`: If specified this program will exit with an
error rather than overwrite a file
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
kwargs = {
'file_in': file_in,
'file_out': destination,
'domain': Domain,
'overwrite': overwrite
}
resp = env.perform_async_action('set_file', **kwargs)
if web:
return util.render_boolean_results_table(
resp, suffix="set_file"), util.render_history(env)
def main(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
base_dir="."):
check_hostname = not no_check_hostname
env = datapower.Environment(
hostnames=appliances,
credentials=credentials,
timeout=timeout,
check_hostname=check_hostname)
for appliance in env.appliances:
_dir = os.path.join(base_dir, appliance.hostname)
if not os.path.exists(_dir):
os.makedirs(_dir)
for domain in appliance.domains:
if domain == "default":
remote_name = "config:///autoconfig.cfg"
local_name = os.path.join(_dir, "autoconfig.cfg")
with open(local_name, "w") as fout:
contents = appliance.getfile(domain=domain, filename=remote_name)
fout.write(contents)
else:
remote_name = "config:///{}.cfg".format(domain)
local_name = os.path.join(_dir, "{}.cfg".format(domain))
with open(local_name, "w") as fout:
contents = appliance.getfile(domain=domain, filename=remote_name)
fout.write(contents)
def flush_aaa_cache(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
Domain="",
aaa_policy="",
web=False):
"""Flushes the AAA Cache of the specified aaa_policy in the
specified Domain.
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-D, --Domain`: The domain where the aaa_policy resides
* `-A, --aaa-policy`: the AAAPolicy who's cache you would like to flush
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
logger = make_logger("mast.accounts")
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
msg = "Attempting to flush AAA cache on {}".format(str(env.appliances))
logger.info(msg)
if not web:
print msg
kwargs = {"PolicyName": aaa_policy, 'domain': Domain}
responses = env.perform_action('FlushAAACache', **kwargs)
logger.debug("Responses received {}".format(str(responses)))
if not web:
for host, resp in responses.items():
print "{}\n{}".format(host, "=" * len(host))
pprint_xml(resp.xml)
else:
return util.render_boolean_results_table(
responses, suffix="flush_aaa_cache"), util.render_history(env)
def main(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
domains=[],
out_file="tmp/conformance-validation.xlsx"):
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout=timeout,
check_hostname=check_hostname)
profiles = [
"dp-wsi-bp.xsl",
"dp-cfg-bp.xsl",
"dp-wsi-bsp-1.0.xsl",
]
fields = [
"domain", "profile", "severity", "type", "specification",
"object-type", "object-name", "parameter-name", "permitted-setting",
"actual-setting", "details"
]
wb = openpyxl.Workbook()
ws = wb.active
wb.remove_sheet(ws)
for appliance in env.appliances:
ws = wb.create_sheet()
ws.title = appliance.hostname
ws.append(fields)
print appliance.hostname
_domains = domains
if "all-domains" in domains:
_domains = appliance.domains
for domain in _domains:
print "\t", domain
for profile in profiles:
print "\t\t", profile
appliance.request.clear()
appliance.request.request(
domain=domain).get_conformance_report(profile=profile)
resp = appliance.send_request()
reports = resp.xml.findall(".//Report")
for report in reports:
ws.append([
domain,
profile,
report.get("severity", ""),
report.get("type", ""),
report.get("specification", ""),
report.find('Location').get("object-type", ""),
report.find('Location').get("object-name", ""),
getattr(report.find('ParameterName'), "text", ""),
getattr(report.find('PermittedSetting'), "text", ""),
getattr(report.find('ActualSetting'), "text", ""),
getattr(report.find('Details'), "text", ""),
])
wb.save(out_file)
print "Done"
def main(appliances=[],
credentials=[],
domains=[],
no_check_hostname=False,
max_sessions=20,
timeout=120):
logger = make_logger("session_mon")
check_hostname = not no_check_hostname
env = datapower.Environment(
appliances,
credentials,
timeout,
check_hostname=check_hostname)
if not domains:
domains = ["all-domains"]
for appl in env.appliances:
print(appl.hostname)
response = appl.get_status("ActiveUsers")
_domains = domains
if "all-domains" in domains:
_domains = appl.domains
for domain in _domains:
print("\t{}".format(domain))
sessions = find_sessions(response_xml=response.xml)
logger.debug(
"{} total active sessions found on {}".format(
len(sessions), appl.hostname
)
)
users = get_users_with_sessions(sessions, domain)
for user in users:
print("\t\t{}".format(user))
users_sessions = get_users_session_ids(sessions, user)
logger.debug(
"\t\t\tuser {} found with {} active sessions".format(
user, len(users_sessions)
)
)
print(
"\t\t\tuser {} found with {} active sessions".format(
user, len(users_sessions)
)
)
if len(users_sessions) > max_sessions:
logger.info(
"user {} found with {} active sessions, this is greater than max, disconnecting.".format(
user, len(users_sessions)
)
)
print(
"\t\t\tuser {} found with {} active sessions, this is greater than max, disconnecting.".format(
user, len(users_sessions)
)
)
disconnect_all(appl, user, users_sessions)
def list_checkpoints(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
Domain="",
web=False):
"""Lists the checkpoints which are currently in the
specified domain
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-D, --Domain`: The domain to list the checkpoints for
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
logger = make_logger("mast.backups")
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
logger.info("Attempting to list checkpoints for {} in {} domain".format(
str(env.appliances), Domain))
resp = env.perform_action("get_existing_checkpoints", domain=Domain)
logger.debug("Responses received: {}".format(str(resp)))
if web:
return (util.web_list_checkpoints(resp,
Domain), util.render_history(env))
for host, d in resp.items():
print host, '\n', '=' * len(host)
for key, value in d.items():
print key, "-".join(value["date"]), ":".join(value["time"])
print
def main(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
domains=[],
paths=[],
force=False,
dry_run=False):
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout=timeout,
check_hostname=check_hostname)
for appliance in env.appliances:
_domains = domains
if "all-domains" in _domains:
_domains = appliance.domains
print(appliance.hostname)
for domain in domains:
print("\t{}".format(domain))
filestore_dict = defaultdict(list)
for path in paths:
location = path.split(":")[0] + ":"
if location not in filestore_dict:
filestore = appliance.get_filestore(domain, location)
filestore_dict[location] = etree.fromstring(filestore.text)
for directory in filestore_dict[location].xpath(".//directory"):
if fnmatch(directory.get("name"), path):
print("\t\t{}".format(directory.get("name")))
if len(directory) and not force:
print("\t\t\tDirectory not empty, use `--force` to remove this directory")
continue
if dry_run:
continue
resp = appliance.RemoveDir(
domain=domain,
Dir=directory.get("name"),
)
resp = etree.fromstring(resp.text)
msg = " ".join(resp.itertext())
print("\t\t\t{}".format(msg))
continue
for node in directory.xpath("./file"):
name = "/".join((node.getparent().get("name"), node.get("name")))
if fnmatch(name, path):
print("\t\t{}".format(name))
if dry_run:
continue
resp = appliance.DeleteFile(
domain=domain,
File=name
)
resp = etree.fromstring(resp.text)
msg = " ".join(resp.itertext())
print("\t\t\t{}".format(msg))
def main(
appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
dry_run=False,
save_config=False,
):
check_hostname = not no_check_hostname
env = datapower.Environment(hostnames=appliances,
credentials=credentials,
timeout=timeout,
check_hostname=check_hostname)
allaterror = etree.fromstring(
'<LogEvents><Class class="LogLabel">all</Class><Priority>error</Priority></LogEvents>'
)
for appliance in env.appliances:
print(appliance.hostname)
for domain in appliance.domains:
changed = False
print("\t{}".format(domain))
config = appliance.get_config("LogTarget",
"default-log",
domain=domain,
persisted=False)
config = etree.fromstring(str(config))
#config = config.xpath(r"/env:Envelope/env:Body/dp:response/dp:config/LogTarget", nsmap=nsmap)
config = config.xpath(
r'/*[local-name()="Envelope"]/*[local-name()="Body"]/*[local-name()="response"]/*[local-name()="config"]/*[local-name()="LogTarget"]'
)[0]
if len(
config.xpath(
'./LogEvents[not(./Class/text()="all") or not(./Priority/text()="error")]'
)):
changed = True
#pprint_xml(config)
for nodeset in config.xpath(r'./*[local-name()="LogEvents"]'):
config.remove(nodeset)
config.append(allaterror)
#pprint_xml_str(etree.tostring(config))
appliance.request.clear()
request = appliance.request.request(domain=domain).set_config
request.append(config)
for nodeset in config.xpath(r'.//*'):
if "read-only" in nodeset.attrib:
nodeset.attrib.pop("read-only")
#print(appliance.request)
if not dry_run:
print(appliance.send_request(boolean=True))
else:
pprint_xml(allaterror)
print(" ")
if save_config and not dry_run and changed:
appliance.SaveConfig(domain=domain)
def main(
appliances=[],
credentials=[],
domains=[],
no_check_hostname=False,
timeout=120,
object_class="",
commands=[],
dry_run=False,
save_config=False,
):
logger = make_logger("session_mon")
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
for appliance in env.appliances:
print("{}\n================\n\n".format(appliance.hostname))
_domains = domains
if "all-domains" in domains:
_domains = appliance.domains
_commands = []
for domain in _domains:
config = appliance.get_config(
_class=object_class,
domain=domain,
persisted=False,
)
objs = config.xml.findall(datapower.CONFIG_XPATH)
if not objs:
continue
_commands.append("switch domain {}".format(domain))
_commands.append("co")
for obj in objs:
_commands.append("{} {}".format(object_class, obj.get("name")))
for command in commands:
_commands.append(command)
_commands.append("exit")
if save_config:
_commands.append("write mem")
_commands.append("exit")
if dry_run:
for _command in _commands:
print(_command)
else:
appliance.ssh_connect()
for _command in _commands:
print(appliance.ssh_issue_command(_command))
appliance.ssh_disconnect()
def status(self):
logger = make_logger("mast.datapower.status")
t = Timestamp()
check_hostname = "true" in flask.request.form.get(
'check_hostname').lower()
appliances = flask.request.form.getlist('appliances[]')
credentials = [
xordecode(_,
key=xorencode(
flask.request.cookies["9x4h/mmek/j.ahba.ckhafn"]))
for _ in flask.request.form.getlist('credentials[]')
]
if not appliances:
return flask.abort(404)
env = datapower.Environment(appliances,
credentials,
check_hostname=check_hostname)
providers = flask.request.form.getlist("providers[]")
resp = {"appliances": appliances, "time": t.short}
for provider in providers:
_provider = provider.split(".")[0]
resp[provider] = []
for appliance in env.appliances:
try:
_status = appliance.get_status(_provider)
except datapower.AuthenticationFailure:
# This is to handle an intermittent authentication failure
# sometimes issued by the DataPower. We will sleep 2
# seconds and try again
sleep(2)
try:
return self.status()
except:
logger.exception(
"An unhandled exception occurred during execution")
raise
except:
logger.exception(
"An unhandled exception occurred during execution")
raise
metric = _status.xml.find(PROVIDER_MAP[provider]).text
resp[provider].append(metric)
return flask.jsonify(resp)
def main(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
domains=[],
out_file="tmp/cert-audit.json",
object_classes=[],
object_names=[]):
logger = make_logger("find_refs")
if out_file is None:
logger.error("Must specify out file")
sys.exit(2)
if not os.path.exists(os.path.dirname(out_file)):
os.makedirs(os.path.dirname(out_file))
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
out = tree()
for appliance in env.appliances:
hostname = appliance.hostname
print(hostname)
logger.info("Checking appliance {}".format(appliance.hostname))
_domains = domains
if "all-domains" in domains:
_domains = appliance.domains
for domain in _domains:
print("\t{}".format(domain))
logger.info("In domain {}".format(domain))
config = etree.fromstring(str(appliance.get_config(domain=domain)))
for object_class, object_name in splice(object_classes,
object_names):
obj = config.xpath(".//{}[@name='{}']".format(
object_class, object_name))
if not obj:
print("\t\t{}: {} Not Found".format(
object_class, object_name))
continue
print("\t\t{}: {}".format(object_class, object_name))
node = out[hostname][domain]["{}: {}".format(
object_class, object_name)]
recurse_config(config, object_class, object_name, node)
with open(out_file, "wb") as fp:
json.dump(out, fp, indent=4)
def main(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
domains=[],
ignore_unsaved=False,
dry_run=False,
save_config=False):
check_hostname = not no_check_hostname
env = datapower.Environment(
appliances,
credentials,
timeout=timeout,
check_hostname=check_hostname,
)
debug_mode_off = etree.fromstring("<DebugMode>off</DebugMode>")
for appliance in env.appliances:
print appliance.hostname
_domains = domains
if "all-domains" in domains:
_domains = appliance.domains
for domain in _domains:
changed = False
print "\t", domain
config = appliance.get_config(domain=domain, persisted=False)
config = etree.fromstring(str(config))
objs = config.xpath('.//*[./DebugMode/text()="on"]')
if len(objs):
change = True
for obj in objs:
if not ignore_unsaved and obj.find("DebugMode").attrib.get("persisted") == "false":
appliance.request.clear()
request = appliance.request.request(domain=domain).modify_config()[obj.tag](name=obj.attrib.get("name"))
request.append(debug_mode_off)
if not dry_run:
print(appliance.send_request())
else:
print(appliance.request)
if save_config and not dry_run and changed:
appliance.SaveConfig(domain=domain)
def main(
appliances=[],
credentials=[],
domains=[],
no_check_hostname=False,
timeout=120,
):
logger = make_logger("session_mon")
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
for appliance in env.appliances:
print(appliance.hostname)
_domains = domains
if "all-domains" in domains:
_domains = appliance.domains
for domain in _domains:
print("\t{}".format(domain))
config = appliance.get_config(
_class="WSRRSubscription",
domain=domain,
persisted=False,
)
objs = config.xml.findall(datapower.CONFIG_XPATH)
if not objs:
continue
for obj in objs:
print("\t\t{}".format(obj.get("name")))
resp = appliance.WsrrSynchronize(
domain=domain, WSRRSubscription=obj.get("name"))
print("\t\t\t" + "\n\t\t\t{}".join(
resp.xml.find(
".//{http://www.datapower.com/schemas/management}result"
).itertext()))
def restore_normal_backup(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
file_in=None,
Domain="",
source_type="ZIP",
overwrite_files=True,
overwrite_objects=True,
rewrite_local_ip=True,
deployment_policy=None,
import_domain=True,
reset_domain=True,
dry_run=False,
out_dir="tmp",
web=False):
"""Restores a normal backup to the specified appliances and Domains.
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-f, --file-in`: The backup file which will be restored. This must be in the
format specified in source_type
* `-D, --Domain`: The domain to which to restore the backup
* `-s, --source-type`: The type of backup, must be either "ZIP" or "XML"
* `-N, --no-overwrite-files`: Whether to overwrite files when restoring
the backup
* `--no-overwrite-objects`: Whether to overwrite objects when restoring
the backup
* `--no-rewrite-local-ip`: Whether to rewrite the local IP Addresses
* `-d, --deployment-policy`: The deployment policy to apply when restoring
the backup
* `--no-import-domain`: Whether we are importing a domain
* `--no-reset-domain`: Whether to reset the domain
* `--dry-run`: Whether this should be a dry-run
* `-o, --out_dir`: (NOT NEEDED IN WEB GUI) The directory (local) where you would
want all of the files generated by the restore to be placed
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
logger = make_logger("mast.backups")
t = Timestamp()
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
logger.info(
"Attempting to restore normal backup on {} in {} domain".format(
str(env.appliances), Domain))
kwargs = {
"file_in": file_in,
"source_type": source_type,
"domain": Domain,
"overwrite_files": overwrite_files,
"overwrite_objects": overwrite_objects,
"rewrite_local_ip": rewrite_local_ip,
"deployment_policy": deployment_policy,
"import_domain": import_domain,
"reset_domain": reset_domain,
"dry_run": dry_run
}
resp = env.perform_action("restore_normal_backup", **kwargs)
logger.debug("Responses received {}".format(str(resp)))
out_dir = os.path.join(out_dir, "restore_normal_backup", t.timestamp)
os.makedirs(out_dir)
for host, r in resp.items():
filename = os.path.join(
out_dir, "{}-{}-{}-results.xml".format(t.timestamp, host, Domain))
with open(filename, 'wb') as fout:
fout.write(r.pretty)
if web:
return util.render_see_download_table(resp), util.render_history(env)
def clean_up(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
Domain='default',
checkpoints=False,
export=False,
error_reports=False,
recursive=False,
logtemp=False,
logstore=False,
backup_files=True,
out_dir='tmp',
web=False):
"""This will clean up the specified appliances filesystem optionally
(defaults to True) taking copies of the files as backups.
Parameters:
* `-a, --appliances` - The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-D, --Domain`: The domain who's filesystem you would like to clean up
* `-C, --checkpoints`: If specified, all checkpoints will be removed
from the domain
* `-e, --export`: If specified all exports will be removed from the domain
* `-l, --logtemp`: If specified, all files in `logtemp:` will be removed
from the domain
* `-L, --logstore`: If specified, all files in `logstore:` will be
removed
* `-E, --error-reports`: If specified, all error reports will be removed
from the appliance(s)
* `-r, --recursive`: If specified, directories will be cleaned recursively
* `--no-backup-files`: If specified, files will not be backed up before
deleting
* `-o, --out-dir`: The directory to save backed up files
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
t = Timestamp()
dirs = []
if checkpoints:
dirs.append('chkpoints:/')
if export:
dirs.append('export:/')
if logtemp:
dirs.append('logtemp:/')
if logstore:
dirs.append('logstore:/')
if web:
rows = []
for appliance in env.appliances:
if web:
rows.append((appliance.hostname, ))
for _dir in dirs:
_clean_dir(appliance, _dir, Domain, recursive, backup_files,
t.timestamp, out_dir)
if web:
rows.append(("", _dir, "Cleaned"))
if error_reports:
_clean_error_reports(appliance, Domain, backup_files, t.timestamp,
out_dir)
rows.append(("", "ErrorReports", "Cleaned"))
return flask.render_template(
"results_table.html",
header_row=["Appliance", "Location", "Action"],
rows=rows), util.render_history(env)
def restore_secure_backup(appliances=[],
credentials=[],
timeout=1200,
no_check_hostname=False,
CryptoCertificate="",
location="",
validate_only=False,
web=False):
"""Restores a secure backup to the specified appliances.
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-C, --CryptoCertificate`: The CryptoCertificate object with which
the secure backup was encrypted
* `-l, --location`: The location on the appliances where the SecureBackup
resides (This means that you will have to upload the secure backup
if you got it from MAST, external to the appliance)
* `-v, --validate-only`: If specified then the appliances will only attemp to
validate the backup instead of actually restoring it
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
logger = make_logger("mast.backups")
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
logger.info("Attempting to restore Secure Backup on {}".format(
str(env.appliances)))
validate = "on" if validate_only else "off"
kwargs = {
"cred": CryptoCertificate,
"source": location,
"validate": validate
}
resp = env.perform_action("SecureRestore", **kwargs)
logger.debug("Responses received: {}".format(str(resp)))
if web:
return (util.render_boolean_results_table(resp),
util.render_history(env))
for host, msg in resp.items():
print host, '\n', "=" * len(host)
print msg
print
def get_secure_backup(appliances=[],
credentials=[],
timeout=1200,
no_check_hostname=False,
out_dir='tmp',
CryptoCertificate="",
destination='local:/raid0',
include_iscsi=False,
include_raid=False,
remove=True,
quiesce_before=True,
unquiesce_after=True,
quiesce_timeout=60,
web=False):
"""Performs a secure backup of the specified domain.
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-o, --out-dir`: (NOT NEEDED IN WEB GUI) The directory (local) to store
the backup
* `-C, --CryptoCertificate`: The CryptoCertificate object to use to encrypt
the backup
* `-d, --destination`: The base location (on the appliance) to store
the backup
* `-i, --include-iscsi`: Whether to include the iscsi filesystem
* `-I, --include-raid`: Whether to include the RAID filesystem
* `-N, --no-remove`: If specified the backup will NOT be removed from
the DataPower
* `--no-quiesce-before`: If specified, the appliance will not be
quiesced before performing the secure backup
* `--no-unquiesce-after`: If specified, the appliance will not be
unquiesced after performing the secure backup
* `-q, --quiesce-timeout`: The timeout to wait before the appliance
attempts to quiesce
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
logger = make_logger("mast.backups")
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
output = ""
if quiesce_before:
resp = {}
for appliance in env.appliances:
logger.info("Quiescing {} in preparation of Secure Backup".format(
appliance.hostname))
resp[appliance.hostname] = appliance.QuiesceDP(
timeout=quiesce_timeout)
logger.debug("Response received {}".format(
resp[appliance.hostname]))
if web:
output += util.render_boolean_results_table(
resp, suffix="Quiesce_appliance")
sleep(quiesce_timeout)
t = Timestamp()
if destination.endswith("/"):
destination = destination.rstrip("/")
destination = '%s/%s' % (destination, t.timestamp)
kwargs = {'Dir': destination, 'domain': 'default'}
logger.info("Creating directory {} on {} to store Secure Backup".format(
destination, str(env.appliances)))
resp = env.perform_async_action('CreateDir', **kwargs)
logger.debug("Responses received {}".format(str(resp)))
if web:
output += util.render_boolean_results_table(resp, suffix="CreateDir")
include_raid = 'on' if include_raid else 'off'
include_iscsi = 'on' if include_iscsi else 'off'
kwargs = {
'cert': CryptoCertificate,
'destination': destination,
'include_iscsi': include_iscsi,
'include_raid': include_raid
}
logger.info("Attempting to perform a Secure Backup on {}".format(
str(env.appliances)))
resp = env.perform_async_action('SecureBackup', **kwargs)
logger.debug("Responses received: {}".format(str(resp)))
if web:
output += util.render_boolean_results_table(resp,
suffix="SecureBackup")
if web:
results = {}
remove_results = {}
for appliance in env.appliances:
directory = os.path.join(out_dir, appliance.hostname, "SecureBackup",
t.timestamp)
start = time()
while not appliance.file_exists(
'{}/backupmanifest.xml'.format(destination), 'default'):
sleep(5)
if time() - start > timeout:
raise TimeoutError
logger.info("Attempting to retrieve Secure Backup from {}".format(
appliance.hostname))
appliance.copy_directory(destination, directory)
_directory = os.path.join(
directory,
destination.replace(":", "").replace("///", "/"))
try:
logger.info("Attempting to verify Secure Backup for {}".format(
appliance.hostname))
if appliance.verify_local_backup(_directory):
logger.info("Secure Backup integrity verified for {}".format(
appliance.hostname))
if web:
results[appliance.hostname] = "Succeeded"
else:
print '\t', appliance.hostname, " - ", "Succeeded"
if remove:
logger.info(
"Attempting to remove Secure Backup from appliance "
"{}".format(appliance.hostname))
_resp = appliance.RemoveDir(Dir=destination,
domain='default')
logger.debug("Response received: {}".format(_resp))
if web:
remove_results[appliance.hostname] = _resp
else:
logger.warn("Secure Backup for {} Corrupt!".format(
appliance.hostname))
if web:
results[appliance.hostname] = "Failed"
else:
print '\t', appliance.hostname, " - ", "Failed"
appliance.log_error('Verification of backup in %s failed' %
(_directory))
except:
if web:
results[appliance.hostname] = "Failed"
logger.exception(
"An unhandled exception occurred during execution.")
if web:
output += util.render_results_table(results,
suffix="verify-SecureBackup")
output += util.render_boolean_results_table(remove_results,
suffix="RemoveDir")
if unquiesce_after:
resp = {}
for appliance in env.appliances:
logger.info("Attempting to unquiesce {}".format(
str(appliance.hostname)))
resp[appliance.hostname] = appliance.UnquiesceDP()
logger.debug("Response received: {}".format(
resp[appliance.hostname]))
if web:
output += util.render_boolean_results_table(
resp, suffix="Unquiesce_appliance")
if web:
return output, util.render_history(env)
def get_normal_backup(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
Domain=[],
comment="",
out_dir='tmp',
individual=False,
web=False):
"""Performs a normal backup of the specified domain.
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-D, --Domain`: The domains to backup (all-domains will backup all domains)
To spcify multiple domains,
use multiple entries of the form `[-D domain1 [-D domain2...]]`
* `-C, --comment`: The comment to add to the backup
* `-o, --out-dir`: (NOT NEEDED IN WEB GUI) The directory (local) where you
would like to store the backup
* `-I, --individual`: If specified and all-domains is specified as --Domain
then backup each domain individually instead of "all-domains"
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
logger = make_logger("mast.backups")
t = Timestamp()
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
if not Domain:
raise ValueError(
"Must provide one or more domains including 'all-domains'")
if isinstance(Domain, basestring):
Domain = [Domain]
# Fixes duplicate domains issue
Domain = list(set(Domain))
results = {}
if not individual:
logger.info("Attempting to retrieve normal backup from "
"{} in {} domain".format(str(env.appliances), Domain))
kwargs = {'domains': Domain, 'comment': comment}
_results = env.perform_async_action('get_normal_backup', **kwargs)
logger.debug("backups retrieved, check file for contents")
for hostname, backup in _results.items():
directory = os.path.join(out_dir, hostname, "NormalBackup",
t.timestamp)
os.makedirs(directory)
filename = os.path.join(
directory,
'%s-%s-%s.zip' % (t.timestamp, hostname, "_".join(Domain)))
logger.debug("Writing backup for {} to {}".format(
hostname, filename))
with open(filename, 'wb') as fout:
fout.write(backup)
if _verify_zip(filename):
logger.info("backup for {} in {} domain verified".format(
hostname, str(Domain)))
results[hostname + "-" + "_".join(Domain) +
"-normalBackup"] = "Verified"
else:
logger.info("backup for {} in {} domain corrupt".format(
hostname, str(Domain)))
results[hostname + "-" + "_".join(Domain) +
"-normalBackup"] = "Corrupt"
else:
for domain in Domain:
logger.info("Attempting to retrieve normal backup from "
"{} in {} domain".format(str(env.appliances), domain))
kwargs = {'domains': domain, 'comment': comment}
_results = env.perform_async_action('get_normal_backup', **kwargs)
logger.debug("backups retrieved, check file for contents")
for hostname, backup in _results.items():
directory = os.path.join(out_dir, hostname, "NormalBackup",
t.timestamp)
if not os.path.exists(directory):
os.makedirs(directory)
filename = os.path.join(
directory,
'%s-%s-%s.zip' % (t.timestamp, hostname, domain))
logger.debug("Writing backup for {} to {}".format(
hostname, filename))
with open(filename, 'wb') as fout:
fout.write(backup)
if _verify_zip(filename):
logger.info("backup for {} in {} domain verified".format(
hostname, domain))
results[hostname + "-" + domain +
"-normalBackup"] = "Verified"
else:
logger.info("backup for {} in {} domain corrupt".format(
hostname, domain))
results[hostname + "-" + domain +
"-normalBackup"] = "Corrupt"
if web:
return util.render_results_table(results), util.render_history(env)
for k, v in results.items():
print
print k
print '=' * len(k)
print v
print
def predeploy(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
out_dir="tmp",
Domain="",
comment="",
predeploy_command=None,
CryptoCertificate="",
secure_backup_destination="local:/raid0",
backup_default=True,
backup_all=True,
do_secure_backup=False,
do_normal_backup=True,
set_checkpoints=True,
include_iscsi=False,
include_raid=False,
remove_secure_backup=True,
default_checkpoint=True,
remove_oldest_checkpoint=True,
allow_shell=False,
web=False):
"""Perform routine pre-deployment actions. Everything is optional, but if
you wish to perform an action, you must provide the necessary arguments.
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-o, --out-dir`: (Not needed in web GUI) The directory (local)
where you would like to store the artifacts generated by this
script
* `-D, --Domain`: This is the app domain to backup and the app domain
in which to set a checkpoint
* `-C, --comment`: This is shared among other actions. The comment is
used to build the name of the checkpoint.
* `-p, --predeploy-command`: This command will be "shelled out"
to the machine running MAST after performing the backups and checkpoints.
Use this parameter to pull from version control or similar
operations.
* `--CryptoCertificate`: The CryptoCertificate with which to
encrypt the secure backup
* `-s, --secure-backup-destination`: The destination (on the DataPower)
where the secure backup will be stored
* `-N, --no-backup-default`: Whether to also backup the default domain
* `--no-backup-all`: Whether to also backup all-domains
* `-d, --do-secure-backup`: Whether to retrieve a secure backup
* `--no-do-normal-backup`: Whether to retrieve normal backups
* `--no-set-checkpoints`: Whether to set checkpoints
* `-i, --include-iscsi`: Whether to include the iscsi volume in the
secure backup
* `-I, --include-raid`: Whether to include the raid volume in the
secure backup
* `--no-remove-secure-backup`: Whether to remove the secure backup
from the appliance after verifying your local copy.
* `--no-default-checkpoint`: Whether to create a checkpoint in the
default domain
* `--no-remove-oldest-checkpoint`: Whether to remove the oldest
checkpoint from the domain IF AND ONLY IF the maximum number
of checkpoints has been reached.
* `-A, --allow-shell`: Whether to allocate a shell for the execution of
the predeploy-command
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__
Here is what is possible (will be done in this order):
1. Secure Backup
* The following parameters apply:
* `-d, --do-secure-backup`
* `-o, --out-dir`
* `--CryptoCertificate`
* `-s, --secure-backup-destination`
* `-i, --include-iscsi`
* `-I, --include-raid`
* `--no-remove-secure-backup`
2. Normal Backups
* The following parameters apply:
* `--no-do-normal-backup`
* `-o, --out-dir`
* `-D, --Domain`
* `-N, --no-backup-default`
* `--no-backup-all`
* `-C, --comment`
3. Checkpoints
* The following parameters apply:
* `--no-set-checkpoints`
* `-D, --Domain`
* `-C, --comment`
* `--no-default-checkpoint`
* `--no-remove-oldest-checkpoint`"""
from mast.datapower.backups import set_checkpoint
from mast.datapower.backups import get_normal_backup, get_secure_backup
logger = make_logger('mast.datapower.deployment')
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
if web:
output = ""
history = ""
# Loop through appliances so we will only affect one appliance at a time
for appliance in env.appliances:
# Secure Backup
if do_secure_backup:
if not CryptoCertificate:
# Fail if CryptoCertificate is not given
logger.error("Cert must be specified in order "
"to perform a secure backup!")
sys.exit(-1)
logger.info("Starting Secure Backup for {}".format(
appliance.hostname))
_out = get_secure_backup(appliances=appliance.hostname,
credentials=appliance.credentials,
timeout=timeout,
out_dir=out_dir,
CryptoCertificate=CryptoCertificate,
destination=secure_backup_destination,
include_iscsi=include_iscsi,
include_raid=include_raid,
remove=remove_secure_backup,
quiesce_before=False,
unquiesce_after=False,
no_check_hostname=no_check_hostname,
web=web)
if web:
output += _out[0]
history += _out[1]
logger.info("Finished Secure Backup for {}".format(
appliance.hostname))
# Normal backups
if do_normal_backup:
logger.info("Pre-Deployment backups started at {}".format(
str(Timestamp())))
domains = [Domain]
if backup_default:
domains.append("default")
if backup_all:
domains.append("all-domains")
_out = get_normal_backup(appliances=[appliance.hostname],
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
Domain=domains,
comment=comment,
out_dir=out_dir,
web=web)
logger.info("Pre-Deployment backups finished at {}".format(
str(Timestamp())))
if web:
output += _out[0]
history += _out[1]
# Checkpoints
if set_checkpoints:
logger.info("Pre-Deployment checkpoints started at {}".format(
str(Timestamp())))
domains = [Domain]
if default_checkpoint:
domains.append("default")
_out = set_checkpoint(appliances=[appliance.hostname],
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
Domain=domains,
comment=comment,
remove_oldest=remove_oldest_checkpoint,
web=web)
logger.info("Pre-Deployment checkpoints finished at {}".format(
str(Timestamp())))
if web:
output += _out[0]
history += _out[1]
if predeploy_command:
logger.info(
"Pre-Deployment command '{}' found. Executing at {}".format(
predeploy_command, str(Timestamp())))
out, err = system_call(command=predeploy_command, shell=allow_shell)
out = str(out)
err = str(err)
logger.info(
"finished executing Pre-Deployment command '{}' at {}., output: {}"
.format(predeploy_command, str(Timestamp()), ";".join([out, err])))
if web:
from mast.plugin_utils.plugin_utils import render_results_table
results = {
"predeploy command":
"{}\n\nout: {}\n\nerr: {}".format(predeploy_command, out, err)
}
output += render_results_table(results)
else:
print "Finished running pre-deploy command. output: {}".format(
";".join([out, err]))
if web:
return output, history
def postdeploy(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
Domain="",
unquiesce_domain=True,
unquiesce_appliance=False,
postdeploy_command=None,
save_config=True,
web=False):
"""This is a simple script which will allow you to unquiesce
your domain or appliances after you quiesce them for a deployment.
Also this will allow you to save the config.
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-D, --Domain`: The domain which will be unquiesced and persisted.
* `-N, --no-unquiesce-domain`: If specified, this script will not attempt
to unquiesce the domain
* `-u, --unquiesce-appliance`: If specified, this script will attempt to
unquiesce the appliance
* `-p, --postdeploy-command`: This command will be "shelled out"
to the machine running MAST after unquiescing and saving the configuration,
use this parameter to clean up VCS artifacts and/or perform check-outs
of your deployed services or similar operations
* `--no-save-config`: If specified, the configuration will not be saved in
the application domain
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
import mast.datapower.system as system
check_hostname = not no_check_hostname
logger = make_logger('mast.datapower.deployment')
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
if web:
output = ""
history = ""
for appliance in env.appliances:
if unquiesce_appliance:
appliance.log_info("Attempting to unquiesce appliance")
_out = system.unquiesce_appliance(
appliances=[appliance.hostname],
credentials=[appliance.credentials],
timeout=timeout,
no_check_hostname=no_check_hostname,
web=web)
appliance.log_info("Finished Unquiescing appliance")
if web:
output += _out[0]
history += _out[1]
else:
print "Finished unquiescing appliance"
if unquiesce_domain:
appliance.log_info("Attempting to unquiesce domain")
_out = system.unquiesce_domain(appliances=[appliance.hostname],
credentials=[appliance.credentials],
timeout=timeout,
no_check_hostname=no_check_hostname,
Domain=Domain,
web=web)
appliance.log_info("Finished Unquiescing domain")
if web:
output += _out[0]
history += _out[1]
else:
print "Finished unquiescing domain"
if save_config:
appliance.log_info(
"Attempting to save configuration after deployment")
_out = system.save_config(appliances=[appliance.hostname],
credentials=[appliance.credentials],
timeout=timeout,
no_check_hostname=no_check_hostname,
Domain=Domain,
web=web)
appliance.log_info(
"Finished saving configuration after deployment")
if web:
output += _out[0]
history += _out[1]
else:
print "Finished saving the configuration"
if postdeploy_command:
logger.info(
"Post-Deployment command '{}' found. Executing at {}".format(
postdeploy_command, str(Timestamp())))
out, err = system_call(command=postdeploy_command)
out = str(out)
err = str(err)
logger.info(
"finished executing Post-Deployment command '{}' at {}., output: {}"
.format(postdeploy_command, str(Timestamp()), ";".join([out,
err])))
if web:
from mast.plugin_utils.plugin_utils import render_results_table
results = {
"postdeploy command":
"{}\n\nout: {}\n\nerr: {}".format(postdeploy_command, out, err)
}
output += render_results_table(results)
else:
print "Finished running post-deploy command. output: {}".format(
";".join([out, err]))
if web:
return output, history
def cert_file_audit(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
out_file=os.path.join("tmp", "cert-file-audit.xlsx")):
locations = ["cert:", "pubcert:", "sharedcert:"]
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
header_row = ["appliance",
"domain",
"filename",
"base-filename",
"size",
"modified"]
rows = []
for appliance in env.appliances:
print appliance.hostname
domain = "default"
for location in locations:
print "\t{}".format(location)
filestore = appliance.get_filestore(domain=domain,
location=location)
_location = filestore.xml.find(datapower.FILESTORE_XPATH)
if _location is None:
continue
if _location.findall("./file") is not None:
for _file in _location.findall("./file"):
dir_name = _location.get("name")
filename = _file.get("name")
filename = "/".join((dir_name, filename))
filename = re.sub(r":[/]*", ":///", filename)
base_filename = filename.split("/")[-1]
print "\t\t{}".format(filename)
size = _file.find("size").text
modified = _file.find("modified").text
rows.append([appliance.hostname,
domain,
filename,
base_filename,
size,
modified])
for directory in _location.findall(".//directory"):
dir_name = directory.get("name")
print "\t\t{}".format(dir_name)
for _file in directory.findall(".//file"):
filename = _file.get("name")
filename = "/".join((dir_name, filename))
filename = re.sub(r":[/]*", ":///", filename)
base_filename = filename.split("/")[-1]
# Infer domain from second segment of path if in cert:
_domain = domain
if location == "cert:":
_domain = filename.replace("///", "/").split("/")[1]
filename = filename.replace(_domain+"/", "")
print "\t\t\t{}".format(filename)
size = _file.find("size").text
modified = _file.find("modified").text
rows.append([appliance.hostname,
_domain,
filename,
base_filename,
size,
modified])
wb = openpyxl.Workbook()
ws = wb.get_active_sheet()
ws.title = "CertFileAudit"
ws.append(header_row)
for row in rows:
ws.append(row)
wb.save(out_file)
def cert_audit(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
domains=[],
out_file="tmp/cert-audit.xlsx",
delay=0.5):
check_hostname = not no_check_hostname
env = datapower.Environment(
appliances,
credentials,
timeout,
check_hostname=check_hostname)
wb = openpyxl.Workbook()
ws = wb.get_active_sheet()
header_row = [
"appliance",
"domain",
"certificate-object",
"filename",
"subject",
"SANs",
"not-before",
"not-after",
"issuer"]
ws.append(header_row)
for appliance in env.appliances:
logger.info("Checking appliance {}".format(appliance.hostname))
print appliance.hostname
_domains = domains
if "all-domains" in domains:
_domains = appliance.domains
for domain in _domains:
logger.info("In domain {}".format(domain))
print "\t", domain
config = appliance.get_config("CryptoCertificate", domain=domain)
certs = [x for x in config.xml.findall(datapower.CONFIG_XPATH)]
# Filter out disabled objects because the results won't change,
# but we will perform less network traffic
certs = filter(
lambda x: x.find("mAdminState").text == "enabled",
certs)
for cert in certs:
logger.info("Exporting cert {}".format(cert))
filename = cert.find("Filename").text
filename = re.sub(r":[/]*", ":///", filename)
name = cert.get("name")
_filename = name
print "\t\t", name
row = [appliance.hostname, domain, name, filename]
appliance.CryptoExport(
domain=domain,
ObjectType="cert",
ObjectName=name,
OutputFilename=_filename)
logger.info("Finished exporting cert {}".format(cert))
try:
logger.info(
"Retrieving file {}".format(
"temporary:///{}".format(_filename)))
cert = appliance.getfile(
domain,
"temporary:///{}".format(_filename))
logger.info(
"Finished retrieving file {}".format(
"temporary:///{}".format(_filename)))
logger.info(
"Attempting to delete file {}".format(
"temporary:///{}".format(_filename)))
appliance.DeleteFile(
domain=domain,
File="temporary:///{}".format(_filename))
logger.info(
"Finished deleting file {}".format(
"temporary:///{}".format(_filename)))
except UnboundLocalError:
# This most likely means the file did not exist for
# some reason (skip)
continue
except:
logger.exception("An unhandled exception has occurred")
print appliance.history
print "SKIPPING CERT"
continue
cert = etree.fromstring(cert)
_contents = insert_newlines(cert.find("certificate").text)
certificate = \
"-----BEGIN CERTIFICATE-----\n" +\
_contents +\
"\n-----END CERTIFICATE-----\n"
_cert = OpenSSL.crypto.load_certificate(
OpenSSL.crypto.FILETYPE_PEM,
certificate)
subject = "'{}'".format(
";".join(
["=".join(x)
for x in _cert.get_subject().get_components()]))
sans = []
ext_count = _cert.get_extension_count()
for i in range(0, ext_count):
ext = _cert.get_extension(i)
if 'subjectAltName' in str(ext.get_short_name()):
sans.append(ext.__str__())
sans = "\n".join(sans)
issuer = "'{}'".format(
";".join(
["=".join(x)
for x in _cert.get_issuer().get_components()]))
row.extend(
[subject,
sans,
_cert.get_notBefore(),
_cert.get_notAfter(),
issuer])
ws.append(row)
sleep(delay)
wb.save(out_file)
def main(
appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
domains=[],
object_classes=[],
persisted=False,
recursive=False,
out_file="tmp/get-config.xlsx",
seperator="\n",
obfuscate_passwords=False,
):
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout=timeout,
check_hostname=check_hostname)
worksheets = OrderedDefaultDict(lambda: [
["appliance", "domain", "name"],
])
if not object_classes:
object_classes = [None]
for appliance in env.appliances:
print appliance.hostname
_domains = domains
if "all-domains" in _domains:
_domains = appliance.domains
for domain in _domains:
print "\t{}".format(domain)
for object_class in object_classes:
config = etree.fromstring(
str(
appliance.get_config(domain=domain,
_class=object_class,
recursive=recursive,
persisted=persisted)))
object_status = etree.fromstring(
str(appliance.get_status("ObjectStatus", domain=domain)))
# First sheet should be ServicesStatusPlus
services_status = etree.fromstring(
str(
appliance.get_status("ServicesStatusPlus",
domain=domain)))
sheet = worksheets["ServicesStatusPlus"]
sheet[0] = [
"appliance",
"domain",
"ServiceName",
"ServiceClass",
"LocalIP",
"LocalPort",
"FshClass",
"FshName",
"FshStatus",
"GatewayStatus",
"Summary",
"BackendUrl",
"BackendHostName",
"BackendPort",
"FshDirectory",
"FshGetQ",
"FshTopic",
"FshTopicSelection",
"FshRemoteServer",
"FshRemotePort",
"EnableHTTP",
"HTTPPort",
"EnableHTTPS",
"HTTPSPort",
"PrimaryInterface",
"JunctionType",
"JunctionTypeStandard",
"JunctionTypeVirtual",
"UnresolvedLocalAddress",
]
for node in services_status.findall(datapower.STATUS_XPATH):
sheet.append([
appliance.hostname,
domain,
[{
"hostname": appliance.hostname,
"domain": domain,
"class": node.find("ServiceClass").text,
"name": node.find("ServiceName").text,
}],
node.find("ServiceClass").text,
node.find("LocalIP").text,
node.find("LocalPort").text,
node.find("FshClass").text,
node.find("FshName").text,
node.find("FshStatus").text,
node.find("GatewayStatus").text,
node.find("Summary").text,
node.find("BackendUrl").text,
node.find("BackendHostName").text,
node.find("BackendPort").text,
node.find("FshDirectory").text,
node.find("FshGetQ").text,
node.find("FshTopic").text,
node.find("FshTopicSelection").text,
node.find("FshRemoteServer").text,
node.find("FshRemotePort").text,
node.find("EnableHTTP").text,
node.find("HTTPPort").text,
node.find("EnableHTTPS").text,
node.find("HTTPSPort").text,
node.find("PrimaryInterface").text,
node.find("JunctionType").text,
node.find("JunctionTypeStandard").text,
node.find("JunctionTypeVirtual").text,
node.find("UnresolvedLocalAddress").text,
])
for node in config.findall(datapower.CONFIG_XPATH):
object_class = node.tag
name = node.get("name")
print "\t\t{} - {}".format(object_class, name)
append_row(worksheets[object_class],
appliance.hostname,
domain,
node,
object_status,
seperator=seperator,
obfuscate_passwords=obfuscate_passwords)
if out_file.endswith(".xlsx"):
workbook = openpyxl.Workbook()
workbook.remove_sheet(workbook.active)
for title, data in worksheets.items():
if len(title) > 31:
title = "...".join((title[:14], title[-14:]))
worksheet = workbook.create_sheet(title=title)
for row in data:
for index, cell in enumerate(list(row)):
if isinstance(cell, list):
row[index] = seperator.join(
(item["name"] for item in cell))
worksheet.append(row)
workbook.save(out_file)
elif out_file.endswith(".sqlite"):
con = sqlite3.connect(out_file)
for name, data in worksheets.items():
for row_index, row in enumerate(list(data)):
for cell_index, cell in enumerate(list(row)):
if isinstance(cell, list):
data[row_index][cell_index] = seperator.join(
(item["name"] for item in cell))
df = pd.DataFrame(data, columns=data.pop(0))
df.to_sql(name=name, con=con, if_exists="replace", index=0)
elif out_file.endswith(".html"):
html = """<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" />
<link rel="stylesheet" href="https://cdn.datatables.net/1.10.19/css/jquery.dataTables.min.css" />
</head>
<body>
<style>
tfoot {
display: table-header-group;
}
</style>
<!-- <div class="container mx-auto"> -->
<ul id="tabs" class="nav nav-tabs" role="tablist" style="display: none;">
"""
for title in sorted(worksheets.keys()):
html += """<li class="nav-item">
<a class="nav-link" role="tab" data-toggle="tab" href="#{0}">{0}</a>
</li>
""".format(title)
html += "</ul>"
html += """<form><select id="tab-select">"""
for index, title in enumerate(sorted(worksheets.keys())):
if title == "ServicesStatusPlus":
html += """<option value="{}" selected="selected">{}</option>""".format(
index, title)
else:
html += """<option value="{}">{}</option>""".format(
index, title)
html += "</select></form>"
html += """<div class="tab-content" id="nav-tabContent">"""
for index, title in enumerate(sorted(worksheets.keys())):
data = worksheets[title]
if title == "ServicesStatusPlus":
html += """<div class="tab-pane fade show active" id="{0}" role="tabpanel"><table class="display table table-striped">""".format(
title)
else:
html += """<div class="tab-pane fade" id="{0}" role="tabpanel"><table class="display table table-striped">""".format(
title)
header = data.pop(0)
html += "<thead><tr>"
for cell in header:
html += """<th>{}</th>""".format(cell)
html += "</tr></thead>"
html += "<tfoot><tr>"
for cell in header:
html += """<th></th>"""
html += "</tr></tfoot><tbody>"
for row in data:
if isinstance(row[2], list):
html += """<tr id="{}-{}-{}-{}">""".format(
row[0].replace(" ", "-"), row[1].replace(" ", "-"),
title.replace(" ", "-"),
row[2][0]["name"].replace(" ", "-"))
else:
html += """<tr id="{}-{}-{}-{}">""".format(
row[0].replace(" ", "-"), row[1].replace(" ", "-"),
title.replace(" ", "-"), row[2].replace(" ", "-"))
for cell in row:
if isinstance(cell, list):
html += "<td>"
for item in cell:
link = "#{}-{}-{}-{}".format(
item["hostname"].replace(" ", "-"),
item["domain"].replace(" ", "-"),
item["class"].replace(" ", "-"),
item["name"].replace(" ", "-"),
)
html += """<a href="{}" class="link" data-class="{}" >{}</a><br/>""".format(
link, item["class"], item["name"])
html += "</td>"
else:
if cell:
html += "<td>{}</td>".format(
cell.replace(seperator, "<br />"))
else:
html += "<td>null</td>"
if len(row) < len(header):
for i in range(len(header) - len(row)):
html += "<td>null</td>"
html += "</tr>"
html += "</tbody></table></div>"
html += """
<!-- </div> -->
</div>
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"></script>
<!-- <script src="https://cdnjs.cloudflare.com/ajax/libs/tether/1.4.0/js/tether.min.js"></script> -->
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js"></script>
<script src="https://cdn.datatables.net/1.10.19/js/jquery.dataTables.min.js"></script>
<script>
$(document).ready(function() {
$('#tab-select').on('change', function (e) {
$('#tabs li a').eq($(this).val()).tab('show');
});
$(".link").on("click", function(e){
e.preventDefault();
var objectClass = $(e.target).attr("data-class");
$(".nav-tabs a[href='#" + objectClass + "']").one("shown.bs.tab", function(){
$($(e.target).attr("href")).css("background-color", "yellow");
$($(e.target).attr("href")).get(0).scrollIntoView();
$('body, html').scrollLeft(0);
});
$("#tab-select").val($("option:contains('" + objectClass + "')").filter(function(){return $(this).text() === objectClass}).val());
$("#tab-select").trigger( "change" );
});
var tables = $('table').DataTable({
paging: false
});
$.each($('table'), function () {
$(this).find('tfoot th').each(function (i) {
var currentTable = $(this).closest('table').DataTable();
var select = $('<input type="text">')
.appendTo($(this))
.on('change', function () {
var val = $(this).val();
console.log(val ? $(this).val() : val)
currentTable.column(i)
.search(val ? $(this).val() : val, false, true)
.draw();
});
});
})
} );
</script>
</body>
</html>"""
with open(out_file, "w") as fp:
fp.write(html)
else:
raise NotImplementedError(
"Unsopported output type, choose one of ['.xlsx', '.sqlite3', '.html']"
)
def list_probes(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
Domain=[],
web=False):
"""Lists all enabled probes in all specified domains
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-D, --Domain`: One or more domains to inspect. To spcify multiple domains,
use multiple entries of the form `[-D domain1 [-D domain2...]]`
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
import urllib2
check_hostname = not no_check_hostname
env = datapower.Environment(
appliances,
credentials,
timeout=timeout,
check_hostname=check_hostname)
results = {}
for appliance in env.appliances:
domains = Domain
if "all-domains" in Domain:
domains = appliance.domains
for domain in domains:
try:
config = appliance.get_config(
_class="all-classes",
name="all-objects",
domain=domain,
persisted=False)
except urllib2.HTTPError:
config = appliance.get_config(domain=domain, persisted=False)
for obj in config.xml.findall(datapower.CONFIG_XPATH):
if obj.find("DebugMode") is not None:
if obj.find("DebugMode").text == "on":
k = "{}-{}".format(appliance.hostname, domain)
v = "{} - {}".format(obj.tag, obj.get("name"))
if k in results:
results[k].append(v)
else:
results[k] = [v]
if web:
for k, v in results.items():
results[k] = "\n".join(v)
return (
util.render_results_table(results),
util.render_history(env))
else:
for k, v in results.items():
print k, "\n", "-" * len(k)
for item in v:
print "\t{}".format(item)
def export(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
Domain="",
object_name=None,
object_class=None,
comment='',
format='ZIP',
persisted=True,
all_files=True,
referenced_files=True,
referenced_objects=True,
out_dir='tmp',
web=False):
"""Exports a service or object to be used to import into another
domain or appliance
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-D, --Domain`: The domain from which to export service/object
* `-o, --object-name`: The name of the object to export
* `-O, --object-class`: The class of the object to export
* `-C, --comment`: The comment to embed into the export
* `-f, --format`: the format in which to export the configuration. This
can be either "XML" or "ZIP"
* `-N, --no-persisted`: If specified, the running configuration will be
exported as opposed to the persisted configuration
* `--no-all-files`: If specified, the export will not include all files
* `--no-referenced-files`: If specified, the referenced files will not
be included in the export
* `--no-referenced-objects`: If specified, referenced objects will not
be included in the export.
* `--out-dir`: (**NOT NEEDED IN THE WEB GUI**)The directory (local)
in which to save the export
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
logger = make_logger("mast.developer")
t = Timestamp()
if object_name is None or object_class is None:
try:
raise TypeError("Must Provide both object name and object class")
except:
logger.exception("Must Provide both object name and object class")
raise
check_hostname = not no_check_hostname
env = datapower.Environment(
appliances,
credentials,
timeout,
check_hostname=check_hostname)
msg = "Attempting to export {} from {}".format(object_name, str(env.appliances))
logger.info(msg)
kwargs = {
'domain': Domain,
'obj': object_name,
'object_class': object_class,
'comment': comment,
'format': format,
'persisted': persisted,
'all_files': all_files,
'referenced_objects': referenced_objects,
'referenced_files': referenced_files}
results = env.perform_action(
'export',
**kwargs)
for hostname, _export in results.items():
d = os.path.join(out_dir, hostname, t.timestamp)
os.makedirs(d)
extention = format.lower()
filename = os.path.join(d, '%s-%s-%s.%s' % (
t.timestamp,
hostname,
object_name,
extention))
msg = "Writing export of {} from {} to {}".format(object_name, hostname, filename)
logger.debug(msg)
if not web:
print msg
with open(filename, 'wb') as fout:
fout.write(_export)
if web:
return util.render_see_download_table(
results, suffix="export"), util.render_history(env)
def deploy(appliances=[],
credentials=[],
timeout=180,
no_check_hostname=False,
Domain=[],
file_in=None,
deployment_policy="",
dry_run=False,
overwrite_files=True,
overwrite_objects=True,
rewrite_local_ip=True,
object_audit=True,
out_dir='tmp',
format='ZIP',
quiesce_domain=True,
quiesce_appliance=False,
quiesce_timeout=120,
web=False):
"""Perform a deployment/migration of a service/object to an IBM DataPower
appliance. This script will try to perform the deployment/migration in a
manner consistent with best practices.
__WARNING__: There is an inherent security risk involved in this script,
in order to allow the most flexible integration with various
Version Control Systems possible, we allow a pre-deployment
hook and a post-deployment hook which will be "shelled out"
to your operating system. For this reason PLEASE be sure to
run this script (and the MAST Web GUI server) as a user with
appropriate permissions.
DO NOT RUN AS ROOT!!!
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-D, --Domain`: The domain to import the configuration into, also the
domain to quiesce if `--no-quiesce-domain` is not specified.
* `-f, --file-in`: This is the configuration file that you are
importing. It must be in the format specified by the format
parameter.
* `-d, --deployment-policy`: The deployment policy to apply to the
import, must already exist on the appliance
* `--dry-run`: If specified, a dry-run will be performed instead of
an actual import
* `-N, --no-overwrite-files`: If specified, no files will be over-
written during the import
* `--no-overwrite-objects`: If specified, no objects will be over-
written during import
* `--no-rewrite-local-ip`: If specified local ip addresses will not
be rewritten on import
* `--no-object-audit`: If specified, an object audit will not be
performed. An object audit is a diff between the running and
persisted configuration
* `-o, --out-dir`: This is where to place the artifacts generated
by this script
* `-F, --format`: The format of the configuration file, must be
either "ZIP" or "XML".
* `--no-quiesce-domain`: If specified, the domain will not be quiesced
prior to the deployment.
* `-q, --quiesce-appliance`: If specified, the appliance will be
quiesced prior to the deployment
* `-Q, --quiesce-timeout`: This is the amount of time for the appliance
to wait before beginning the quiescence procedure.
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
import mast.datapower.system as system
from mast.datapower.developer import _import
logger = make_logger('mast.datapower.deployment')
if web:
output = ""
history = ""
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
for appliance in env.appliances:
appliance.log_info("Deployment started on {}".format(
appliance.hostname))
# Quiesce Domain
if quiesce_domain:
logger.info("Quiescing domain {} before deployment at {}".format(
Domain, str(Timestamp())))
_out = system.quiesce_domain(appliances=[appliance.hostname],
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
Domain=Domain,
quiesce_timeout=quiesce_timeout,
web=web)
logger.info(
"Finished quiescing domain {} before deployment at {}".format(
Domain, str(Timestamp())))
sleep(quiesce_timeout)
if web:
output += _out[0]
history += _out[1]
# Quiesce Appliance
if quiesce_appliance:
logger.info("Quiescing appliances before deployment at {}".format(
str(Timestamp())))
_out = system.quiesce_appliance(
appliances=appliance.hostname,
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
quiesce_timeout=quiesce_timeout,
web=web)
logger.info(
"Finished quiescing appliances before deployment at {}".format(
str(Timestamp())))
sleep(quiesce_timeout)
if web:
output += _out[0]
history += _out[1]
appliance.log_info("Attempting to import configuration at '{}'".format(
str(Timestamp())))
file_out = os.path.join(
out_dir, '{}-deployment_results.txt'.format(appliance.hostname))
# import configuration
_out = _import(appliances=[appliance.hostname],
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
Domain=Domain,
file_in=file_in,
deployment_policy=deployment_policy,
dry_run=dry_run,
overwrite_files=overwrite_files,
overwrite_objects=overwrite_objects,
rewrite_local_ip=rewrite_local_ip,
source_type=format,
out_dir=out_dir,
web=web)
if web:
output += _out[0]
history += _out[1]
appliance.log_info("Finished importing configuration at {}".format(
str(Timestamp())))
# unquiesce domain
if quiesce_domain:
appliance.log_info("Attempting to unquiesce domain")
_out = system.unquiesce_domain(appliances=[appliance.hostname],
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
Domain=Domain,
web=web)
appliance.log_info("Finished unquiescing domain")
if web:
output += _out[0]
history += _out[1]
# unquiesce appliance
if quiesce_appliance:
logger.info("Quiescing appliances before deployment at {}".format(
str(Timestamp())))
_out = system.unquiesce_appliance(
appliances=[appliance.hostname],
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
web=web)
logger.info(
"Finished quiescing appliances before deployment at {}".format(
str(Timestamp())))
if web:
output += _out[0]
history += _out[1]
if object_audit:
appliance.log_info(
"Post-Deployment Object audit started at {}".format(
str(Timestamp())))
_out = system.objects_audit(appliances=[appliance.hostname],
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
out_dir=out_dir,
web=web)
appliance.log_info(
"Post-Deployment Object audit finished at {}".format(
str(Timestamp())))
if web:
output += _out[0]
history += _out[1]
if web:
return output, history
def main(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
base_dir=default_base_dir,
comment=default_comment,
persisted=False,
recursive=False,
no_strip_timestamp=False,
page=False):
"""
track_getconfig.py
Description:
Store running or persisted domain configuration in a local git
repository for auditing purposes.
Usage:
:::bash
$ mast contrib/track_getconfig.py --appliances <HOSTNAMES> --credentials <USER:PASS> --base-dir tmp/config
Parameters:
* `-a, --appliances` - The hostname(s), ip addresse(s), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases please see the comments in `hosts.conf` located
in `$MAST_HOME/etc/default`.
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password` and should be provided in a space-seperated list
if multiple are provided. If you would prefer to not use plain-text
passwords, you can use the output of
`$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-b, --base-dir`: The base directory where to store the downloaded
files. Files will actually be stored in a subdirectory of `base_dir`
named after the hostname in the form of `base_dir/<hostname>`
* `-p, --persisted`: If specified, the persisted configuration will
be retrieved as opposed to the running configuration (which is the
default)
* `-r, --recursive`: If specified, the configuration will be retrieved
recursively to the extent of the facilities provided by DataPower
* `-N, --no-strip-timestamp`: If specified, the timestamp will not be
stripped from the XML document, This is done because we are tracking
this information with git and stripping this out allows us to alert
on any changes in the repository as opposed to special-casing the
difference in timestamp.
* `-P, --page`: If specified, page the output when too long to display
at once
"""
base_dir = os.path.abspath(base_dir)
if not os.path.exists(base_dir):
os.makedirs(base_dir)
try:
repo = git.Repo(base_dir)
repo.head()
except NotGitRepository:
print "Initializing git repository"
git.init(base_dir)
git.add(base_dir)
git.commit(base_dir, message="Initial Commit")
except KeyError:
git.add(base_dir)
git.commit(base_dir, message="Initial Commit")
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout=timeout,
check_hostname=check_hostname)
for appliance in env.appliances:
appliance_directory = os.path.join(base_dir, appliance.hostname)
if not os.path.exists(appliance_directory):
os.mkdir(appliance_directory)
for domain in appliance.domains:
config = appliance.get_config(domain=domain,
recursive=recursive,
persisted=persisted)
config = config.pretty
if no_strip_timestamp:
pass
else:
config = re.sub(r"^.*?<dp:timestamp>.*?</dp:timestamp>.*?$",
r"",
config,
flags=re.MULTILINE)
filename = os.path.join(appliance_directory,
"{}.xml".format(domain))
with open(filename, "wb") as fout:
fout.write(config)
git.add(base_dir)
print git.status(base_dir)
git.commit(base_dir, message=comment)
tmp = StringIO()
git.show(base_dir, outstream=tmp)
tmp.seek(0)
out = highlight(tmp.read(), DiffLexer(), TerminalFormatter())
if page:
pprint.page(out)
else:
print out
def set_checkpoint(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
Domain=['default'],
comment='',
remove_oldest=True,
web=False):
"""Sets a checkpoint in the given domains on the specified appliances
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-D, --Domain`: Domains to set checkpoints in. To spcify multiple domains,
use multiple entries of the form `[-D domain1 [-D domain2...]]`
* `-C, --comment`: The comment to use for the checkpoint (will also be used to
build the checkpoint name)
* `-N, --no-remove-oldest`: If specified this script will attempt to
remove the oldest checkpoint __IF__ the maximum number of checkpoints exist
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
logger = make_logger("mast.backups")
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout,
check_hostname=check_hostname)
logger.info("Attempting to set checkpoint on {} in {} domain(s)".format(
str(env.appliances), str(Domain)))
t = Timestamp()
if web:
header_row = ("Appliance", "Result")
rows = []
for appliance in env.appliances:
if not web:
print appliance.hostname
_domains = Domain
print Domain
print _domains
if "all-domains" in _domains:
_domains = appliance.domains
print _domains
for domain in _domains:
print domain
if not web:
print "\t", domain
name = '{0}-{1}-{2}'.format(comment, domain, t.timestamp)
logger.debug(
"Attempting to set checkpoint {} on {} in {} domain".format(
name, appliance, domain))
if remove_oldest:
_max = appliance.max_checkpoints(domain)
if len(appliance.get_existing_checkpoints(domain)) >= _max:
logger.info(
"Maximum number of checkpoints for domain "
"{} on {} reached. Removing oldest checkpoint.".format(
domain, appliance.hostname))
_resp = appliance.remove_oldest_checkpoint(domain)
logger.debug("Response received: {}".format(_resp))
kwargs = {'domain': domain, 'ChkName': name}
resp = appliance.SaveCheckpoint(**kwargs)
logger.debug("Response received: {}".format(resp))
if not web:
if resp:
print "\t\tSuccessful"
else:
print "\t\tFailed"
if web:
if resp:
rows.append(
("{}-{}-set_checkpoint".format(appliance.hostname,
domain), "Succeeded"))
else:
rows.append(
("{}-{}-set_checkpoint".format(appliance.hostname,
domain), "Failed"))
if web:
return flask.render_template("results_table.html",
header_row=header_row,
rows=rows), util.render_history(env)
def _import(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
Domain=[],
file_in=None,
deployment_policy=None,
deployment_policy_variables=None,
dry_run=False,
overwrite_files=True,
overwrite_objects=True,
rewrite_local_ip=True,
source_type='ZIP',
out_dir="tmp/",
web=False):
"""Import a service/object into the specified appliances
Parameters:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-D. --Domain`: The domain into which the configuration will be imported
* `-f, --file-in`: The file to import into the specified domain. This
__MUST__ match the format specified in source_type
* `-d, --deployment-policy`: The deployment policy to use for the import
(must already exist on the appliances)
* `--dry-run`: Whether to do a dry-run (nothing will be imported)
* `-N, --no-overwrite-files`: If specified, no files will be overwritten
as part of the import
* `--no-overwrite-objects`: If specified, no objects will be overwritten
as part of the import
* `--no-rewrite-local-ip`: If specified, no local ip addresses will be
rewritten as part of the import
* `-s, --source-type`: The type of file to import. Can be "XML" or "ZIP"
* `-o, --out-dir`: The directory to output artifacts generated by this
script
* `-w, --web`: __For Internel Use Only, will be removed in future versions.
DO NOT USE.__"""
logger = make_logger("mast.developer")
t = Timestamp()
check_hostname = not no_check_hostname
env = datapower.Environment(
appliances,
credentials,
timeout,
check_hostname=check_hostname)
msg = "Attempting to import {} to {}".format(file_in, str(env.appliances))
logger.info(msg)
if not web:
print msg
results = {}
out_dir = os.path.join(out_dir, "import_results", t.timestamp)
os.makedirs(out_dir)
for appliance in env.appliances:
if not web:
print appliance.hostname
results[appliance.hostname] = {}
domains = Domain
if "all-domains" in domains:
domains = appliance.domains
for domain in domains:
if not web:
print "\t", domain
kwargs = {
'domain': domain,
'zip_file': file_in,
'deployment_policy': deployment_policy,
'deployment_policy_variables': deployment_policy_variables,
'dry_run': dry_run,
'overwrite_files': overwrite_files,
'overwrite_objects': overwrite_objects,
'rewrite_local_ip': rewrite_local_ip,
'source_type': source_type}
resp = appliance.do_import(**kwargs)
results[appliance.hostname][domain] = resp
if not web:
pprint_xml(resp.xml)
logger.debug("Response received: {}".format(str(resp)))
filename = os.path.join(
out_dir,
"{}-{}-import_results.xml".format(
appliance.hostname,
domain
)
)
with open(filename, 'wb') as fout:
fout.write(resp.pretty)
if web:
return util.render_see_download_table(
results, suffix="import"), util.render_history(env)
def main(appliances=[],
credentials=[],
timeout=120,
EthernetInterface="",
add=[],
remove=[],
save_config=False):
"""
_Script_: `contrib/ssh-add-static-route.py`
DESCRIPTION:
Adds or removes multiple static routes to an EthernetInterface.
USAGE:
To add two static routes and remove two static routes from two appliances,
this is the form you should use:
```
$ mast contrib/ssh-routes.py --appliances APPL_1 --appliances APPL_2 --credentials user:pass --EthernetInterface INT_NAME --add "xxx.xxx.xxx.xxx/xx xxx.xxx.xxx.xxx x" --add "xxx.xxx.xxx.xxx/xx xxx.xxx.xxx.xxx x" --remove "xxx.xxx.xxx.xxx/xx xxx.xxx.xxx.xxx x" --remove "xxx.xxx.xxx.xxx/xx xxx.xxx.xxx.xxx x"
```
RETURNS
PARAMETERS:
* `-a, --appliances`: The hostname(s), ip address(es), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases, please see the comments in `hosts.conf` located in
`$MAST_HOME/etc/default`. To pass multiple arguments to this parameter,
use multiple entries of the form `[-a appliance1 [-a appliance2...]]`
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password`. To pass multiple credentials to this parameter, use
multiple entries of the form `[-c credential1 [-c credential2...]]`.
When referencing multiple appliances with multiple credentials,
there must be a one-to-one correspondence of credentials to appliances:
`[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]`
If you would prefer to not use plain-text passwords,
you can use the output of `$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-E, --EthernetInterface`: The name of the EthernetInterface to which
to add the static route.
* `-A, --add`: The static route(s) to add, should be in
the form of "ip/cidr gateway metric" Where ip/cidr is the destination
IP address and CIDR mask, gateway is the next-hop router's IP Address
and metric is the numerical weighting of the static route. Multiple
static routes can be added by specifying `--add` with proper
arguments multiple times.
* `-r, --remove`: The static route(s) to remove, should be in
the form of "ip/cidr gateway metric" Where ip/cidr is the destination
IP address and CIDR mask, gateway is the next-hop router's IP Address
and metric is the numerical weighting of the static route. Multiple
static routes can be removed by specifying `--remove` with
proper arguments multiple times.
* `-s, --save-config`: If specified, the configuration will be saved.
**NOTE** static routes specified in `--del` and `--add` must contain
spaces and so must be quoted
"""
del_static_routes = [] if remove is None else remove
add_static_routes = [] if add is None else add
env = datapower.Environment(appliances, credentials, timeout=timeout)
env.perform_action("ssh_connect", domain="default")
atexit.register(env.perform_action, "ssh_disconnect")
for appliance in env.appliances:
issue_command = appliance.ssh_issue_command
print appliance.hostname, "\n"
print issue_command("config")
print issue_command("interface {}".format(EthernetInterface))
print issue_command("show route")
for static_route in del_static_routes:
print issue_command("no ip route {}".format(static_route))
for static_route in add_static_routes:
print issue_command("ip route {}".format(static_route))
print issue_command("exit")
if save_config:
print issue_command("write mem")
print issue_command("y")
print issue_command("show route")
print issue_command("exit")
