def set_file(appliances=[], credentials=[], timeout=120, no_check_hostname=False, file_in=None, destination=None, Domain='default', overwrite=True, web=False): """Uploads a file to the specified appliances Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-f, --file-in`: The path and filename of the file to upload * `-d, --destination`: Should be the path and filename of the file once uploaded to the DataPower **NOTE: file_out should contain the filename ie. local:/test.txt** * `-D, --Domain`: The domain to which to upload the file, * `-N, --no-overwrite`: If specified this program will exit with an error rather than overwrite a file * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) kwargs = { 'file_in': file_in, 'file_out': destination, 'domain': Domain, 'overwrite': overwrite } resp = env.perform_async_action('set_file', **kwargs) if web: return util.render_boolean_results_table( resp, suffix="set_file"), util.render_history(env)
def main(appliances=[], credentials=[], timeout=120, no_check_hostname=False, base_dir="."): check_hostname = not no_check_hostname env = datapower.Environment( hostnames=appliances, credentials=credentials, timeout=timeout, check_hostname=check_hostname) for appliance in env.appliances: _dir = os.path.join(base_dir, appliance.hostname) if not os.path.exists(_dir): os.makedirs(_dir) for domain in appliance.domains: if domain == "default": remote_name = "config:///autoconfig.cfg" local_name = os.path.join(_dir, "autoconfig.cfg") with open(local_name, "w") as fout: contents = appliance.getfile(domain=domain, filename=remote_name) fout.write(contents) else: remote_name = "config:///{}.cfg".format(domain) local_name = os.path.join(_dir, "{}.cfg".format(domain)) with open(local_name, "w") as fout: contents = appliance.getfile(domain=domain, filename=remote_name) fout.write(contents)
def flush_aaa_cache(appliances=[], credentials=[], timeout=120, no_check_hostname=False, Domain="", aaa_policy="", web=False): """Flushes the AAA Cache of the specified aaa_policy in the specified Domain. Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-D, --Domain`: The domain where the aaa_policy resides * `-A, --aaa-policy`: the AAAPolicy who's cache you would like to flush * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" logger = make_logger("mast.accounts") check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) msg = "Attempting to flush AAA cache on {}".format(str(env.appliances)) logger.info(msg) if not web: print msg kwargs = {"PolicyName": aaa_policy, 'domain': Domain} responses = env.perform_action('FlushAAACache', **kwargs) logger.debug("Responses received {}".format(str(responses))) if not web: for host, resp in responses.items(): print "{}\n{}".format(host, "=" * len(host)) pprint_xml(resp.xml) else: return util.render_boolean_results_table( responses, suffix="flush_aaa_cache"), util.render_history(env)
def main(appliances=[], credentials=[], timeout=120, no_check_hostname=False, domains=[], out_file="tmp/conformance-validation.xlsx"): check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout=timeout, check_hostname=check_hostname) profiles = [ "dp-wsi-bp.xsl", "dp-cfg-bp.xsl", "dp-wsi-bsp-1.0.xsl", ] fields = [ "domain", "profile", "severity", "type", "specification", "object-type", "object-name", "parameter-name", "permitted-setting", "actual-setting", "details" ] wb = openpyxl.Workbook() ws = wb.active wb.remove_sheet(ws) for appliance in env.appliances: ws = wb.create_sheet() ws.title = appliance.hostname ws.append(fields) print appliance.hostname _domains = domains if "all-domains" in domains: _domains = appliance.domains for domain in _domains: print "\t", domain for profile in profiles: print "\t\t", profile appliance.request.clear() appliance.request.request( domain=domain).get_conformance_report(profile=profile) resp = appliance.send_request() reports = resp.xml.findall(".//Report") for report in reports: ws.append([ domain, profile, report.get("severity", ""), report.get("type", ""), report.get("specification", ""), report.find('Location').get("object-type", ""), report.find('Location').get("object-name", ""), getattr(report.find('ParameterName'), "text", ""), getattr(report.find('PermittedSetting'), "text", ""), getattr(report.find('ActualSetting'), "text", ""), getattr(report.find('Details'), "text", ""), ]) wb.save(out_file) print "Done"
def main(appliances=[], credentials=[], domains=[], no_check_hostname=False, max_sessions=20, timeout=120): logger = make_logger("session_mon") check_hostname = not no_check_hostname env = datapower.Environment( appliances, credentials, timeout, check_hostname=check_hostname) if not domains: domains = ["all-domains"] for appl in env.appliances: print(appl.hostname) response = appl.get_status("ActiveUsers") _domains = domains if "all-domains" in domains: _domains = appl.domains for domain in _domains: print("\t{}".format(domain)) sessions = find_sessions(response_xml=response.xml) logger.debug( "{} total active sessions found on {}".format( len(sessions), appl.hostname ) ) users = get_users_with_sessions(sessions, domain) for user in users: print("\t\t{}".format(user)) users_sessions = get_users_session_ids(sessions, user) logger.debug( "\t\t\tuser {} found with {} active sessions".format( user, len(users_sessions) ) ) print( "\t\t\tuser {} found with {} active sessions".format( user, len(users_sessions) ) ) if len(users_sessions) > max_sessions: logger.info( "user {} found with {} active sessions, this is greater than max, disconnecting.".format( user, len(users_sessions) ) ) print( "\t\t\tuser {} found with {} active sessions, this is greater than max, disconnecting.".format( user, len(users_sessions) ) ) disconnect_all(appl, user, users_sessions)
def list_checkpoints(appliances=[], credentials=[], timeout=120, no_check_hostname=False, Domain="", web=False): """Lists the checkpoints which are currently in the specified domain Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-D, --Domain`: The domain to list the checkpoints for * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" logger = make_logger("mast.backups") check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) logger.info("Attempting to list checkpoints for {} in {} domain".format( str(env.appliances), Domain)) resp = env.perform_action("get_existing_checkpoints", domain=Domain) logger.debug("Responses received: {}".format(str(resp))) if web: return (util.web_list_checkpoints(resp, Domain), util.render_history(env)) for host, d in resp.items(): print host, '\n', '=' * len(host) for key, value in d.items(): print key, "-".join(value["date"]), ":".join(value["time"]) print
def main(appliances=[], credentials=[], timeout=120, no_check_hostname=False, domains=[], paths=[], force=False, dry_run=False): check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout=timeout, check_hostname=check_hostname) for appliance in env.appliances: _domains = domains if "all-domains" in _domains: _domains = appliance.domains print(appliance.hostname) for domain in domains: print("\t{}".format(domain)) filestore_dict = defaultdict(list) for path in paths: location = path.split(":")[0] + ":" if location not in filestore_dict: filestore = appliance.get_filestore(domain, location) filestore_dict[location] = etree.fromstring(filestore.text) for directory in filestore_dict[location].xpath(".//directory"): if fnmatch(directory.get("name"), path): print("\t\t{}".format(directory.get("name"))) if len(directory) and not force: print("\t\t\tDirectory not empty, use `--force` to remove this directory") continue if dry_run: continue resp = appliance.RemoveDir( domain=domain, Dir=directory.get("name"), ) resp = etree.fromstring(resp.text) msg = " ".join(resp.itertext()) print("\t\t\t{}".format(msg)) continue for node in directory.xpath("./file"): name = "/".join((node.getparent().get("name"), node.get("name"))) if fnmatch(name, path): print("\t\t{}".format(name)) if dry_run: continue resp = appliance.DeleteFile( domain=domain, File=name ) resp = etree.fromstring(resp.text) msg = " ".join(resp.itertext()) print("\t\t\t{}".format(msg))
def main( appliances=[], credentials=[], timeout=120, no_check_hostname=False, dry_run=False, save_config=False, ): check_hostname = not no_check_hostname env = datapower.Environment(hostnames=appliances, credentials=credentials, timeout=timeout, check_hostname=check_hostname) allaterror = etree.fromstring( '<LogEvents><Class class="LogLabel">all</Class><Priority>error</Priority></LogEvents>' ) for appliance in env.appliances: print(appliance.hostname) for domain in appliance.domains: changed = False print("\t{}".format(domain)) config = appliance.get_config("LogTarget", "default-log", domain=domain, persisted=False) config = etree.fromstring(str(config)) #config = config.xpath(r"/env:Envelope/env:Body/dp:response/dp:config/LogTarget", nsmap=nsmap) config = config.xpath( r'/*[local-name()="Envelope"]/*[local-name()="Body"]/*[local-name()="response"]/*[local-name()="config"]/*[local-name()="LogTarget"]' )[0] if len( config.xpath( './LogEvents[not(./Class/text()="all") or not(./Priority/text()="error")]' )): changed = True #pprint_xml(config) for nodeset in config.xpath(r'./*[local-name()="LogEvents"]'): config.remove(nodeset) config.append(allaterror) #pprint_xml_str(etree.tostring(config)) appliance.request.clear() request = appliance.request.request(domain=domain).set_config request.append(config) for nodeset in config.xpath(r'.//*'): if "read-only" in nodeset.attrib: nodeset.attrib.pop("read-only") #print(appliance.request) if not dry_run: print(appliance.send_request(boolean=True)) else: pprint_xml(allaterror) print(" ") if save_config and not dry_run and changed: appliance.SaveConfig(domain=domain)
def main( appliances=[], credentials=[], domains=[], no_check_hostname=False, timeout=120, object_class="", commands=[], dry_run=False, save_config=False, ): logger = make_logger("session_mon") check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) for appliance in env.appliances: print("{}\n================\n\n".format(appliance.hostname)) _domains = domains if "all-domains" in domains: _domains = appliance.domains _commands = [] for domain in _domains: config = appliance.get_config( _class=object_class, domain=domain, persisted=False, ) objs = config.xml.findall(datapower.CONFIG_XPATH) if not objs: continue _commands.append("switch domain {}".format(domain)) _commands.append("co") for obj in objs: _commands.append("{} {}".format(object_class, obj.get("name"))) for command in commands: _commands.append(command) _commands.append("exit") if save_config: _commands.append("write mem") _commands.append("exit") if dry_run: for _command in _commands: print(_command) else: appliance.ssh_connect() for _command in _commands: print(appliance.ssh_issue_command(_command)) appliance.ssh_disconnect()
def status(self): logger = make_logger("mast.datapower.status") t = Timestamp() check_hostname = "true" in flask.request.form.get( 'check_hostname').lower() appliances = flask.request.form.getlist('appliances[]') credentials = [ xordecode(_, key=xorencode( flask.request.cookies["9x4h/mmek/j.ahba.ckhafn"])) for _ in flask.request.form.getlist('credentials[]') ] if not appliances: return flask.abort(404) env = datapower.Environment(appliances, credentials, check_hostname=check_hostname) providers = flask.request.form.getlist("providers[]") resp = {"appliances": appliances, "time": t.short} for provider in providers: _provider = provider.split(".")[0] resp[provider] = [] for appliance in env.appliances: try: _status = appliance.get_status(_provider) except datapower.AuthenticationFailure: # This is to handle an intermittent authentication failure # sometimes issued by the DataPower. We will sleep 2 # seconds and try again sleep(2) try: return self.status() except: logger.exception( "An unhandled exception occurred during execution") raise except: logger.exception( "An unhandled exception occurred during execution") raise metric = _status.xml.find(PROVIDER_MAP[provider]).text resp[provider].append(metric) return flask.jsonify(resp)
def main(appliances=[], credentials=[], timeout=120, no_check_hostname=False, domains=[], out_file="tmp/cert-audit.json", object_classes=[], object_names=[]): logger = make_logger("find_refs") if out_file is None: logger.error("Must specify out file") sys.exit(2) if not os.path.exists(os.path.dirname(out_file)): os.makedirs(os.path.dirname(out_file)) check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) out = tree() for appliance in env.appliances: hostname = appliance.hostname print(hostname) logger.info("Checking appliance {}".format(appliance.hostname)) _domains = domains if "all-domains" in domains: _domains = appliance.domains for domain in _domains: print("\t{}".format(domain)) logger.info("In domain {}".format(domain)) config = etree.fromstring(str(appliance.get_config(domain=domain))) for object_class, object_name in splice(object_classes, object_names): obj = config.xpath(".//{}[@name='{}']".format( object_class, object_name)) if not obj: print("\t\t{}: {} Not Found".format( object_class, object_name)) continue print("\t\t{}: {}".format(object_class, object_name)) node = out[hostname][domain]["{}: {}".format( object_class, object_name)] recurse_config(config, object_class, object_name, node) with open(out_file, "wb") as fp: json.dump(out, fp, indent=4)
def main(appliances=[], credentials=[], timeout=120, no_check_hostname=False, domains=[], ignore_unsaved=False, dry_run=False, save_config=False): check_hostname = not no_check_hostname env = datapower.Environment( appliances, credentials, timeout=timeout, check_hostname=check_hostname, ) debug_mode_off = etree.fromstring("<DebugMode>off</DebugMode>") for appliance in env.appliances: print appliance.hostname _domains = domains if "all-domains" in domains: _domains = appliance.domains for domain in _domains: changed = False print "\t", domain config = appliance.get_config(domain=domain, persisted=False) config = etree.fromstring(str(config)) objs = config.xpath('.//*[./DebugMode/text()="on"]') if len(objs): change = True for obj in objs: if not ignore_unsaved and obj.find("DebugMode").attrib.get("persisted") == "false": appliance.request.clear() request = appliance.request.request(domain=domain).modify_config()[obj.tag](name=obj.attrib.get("name")) request.append(debug_mode_off) if not dry_run: print(appliance.send_request()) else: print(appliance.request) if save_config and not dry_run and changed: appliance.SaveConfig(domain=domain)
def main( appliances=[], credentials=[], domains=[], no_check_hostname=False, timeout=120, ): logger = make_logger("session_mon") check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) for appliance in env.appliances: print(appliance.hostname) _domains = domains if "all-domains" in domains: _domains = appliance.domains for domain in _domains: print("\t{}".format(domain)) config = appliance.get_config( _class="WSRRSubscription", domain=domain, persisted=False, ) objs = config.xml.findall(datapower.CONFIG_XPATH) if not objs: continue for obj in objs: print("\t\t{}".format(obj.get("name"))) resp = appliance.WsrrSynchronize( domain=domain, WSRRSubscription=obj.get("name")) print("\t\t\t" + "\n\t\t\t{}".join( resp.xml.find( ".//{http://www.datapower.com/schemas/management}result" ).itertext()))
def restore_normal_backup(appliances=[], credentials=[], timeout=120, no_check_hostname=False, file_in=None, Domain="", source_type="ZIP", overwrite_files=True, overwrite_objects=True, rewrite_local_ip=True, deployment_policy=None, import_domain=True, reset_domain=True, dry_run=False, out_dir="tmp", web=False): """Restores a normal backup to the specified appliances and Domains. Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-f, --file-in`: The backup file which will be restored. This must be in the format specified in source_type * `-D, --Domain`: The domain to which to restore the backup * `-s, --source-type`: The type of backup, must be either "ZIP" or "XML" * `-N, --no-overwrite-files`: Whether to overwrite files when restoring the backup * `--no-overwrite-objects`: Whether to overwrite objects when restoring the backup * `--no-rewrite-local-ip`: Whether to rewrite the local IP Addresses * `-d, --deployment-policy`: The deployment policy to apply when restoring the backup * `--no-import-domain`: Whether we are importing a domain * `--no-reset-domain`: Whether to reset the domain * `--dry-run`: Whether this should be a dry-run * `-o, --out_dir`: (NOT NEEDED IN WEB GUI) The directory (local) where you would want all of the files generated by the restore to be placed * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" logger = make_logger("mast.backups") t = Timestamp() check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) logger.info( "Attempting to restore normal backup on {} in {} domain".format( str(env.appliances), Domain)) kwargs = { "file_in": file_in, "source_type": source_type, "domain": Domain, "overwrite_files": overwrite_files, "overwrite_objects": overwrite_objects, "rewrite_local_ip": rewrite_local_ip, "deployment_policy": deployment_policy, "import_domain": import_domain, "reset_domain": reset_domain, "dry_run": dry_run } resp = env.perform_action("restore_normal_backup", **kwargs) logger.debug("Responses received {}".format(str(resp))) out_dir = os.path.join(out_dir, "restore_normal_backup", t.timestamp) os.makedirs(out_dir) for host, r in resp.items(): filename = os.path.join( out_dir, "{}-{}-{}-results.xml".format(t.timestamp, host, Domain)) with open(filename, 'wb') as fout: fout.write(r.pretty) if web: return util.render_see_download_table(resp), util.render_history(env)
def clean_up(appliances=[], credentials=[], timeout=120, no_check_hostname=False, Domain='default', checkpoints=False, export=False, error_reports=False, recursive=False, logtemp=False, logstore=False, backup_files=True, out_dir='tmp', web=False): """This will clean up the specified appliances filesystem optionally (defaults to True) taking copies of the files as backups. Parameters: * `-a, --appliances` - The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-D, --Domain`: The domain who's filesystem you would like to clean up * `-C, --checkpoints`: If specified, all checkpoints will be removed from the domain * `-e, --export`: If specified all exports will be removed from the domain * `-l, --logtemp`: If specified, all files in `logtemp:` will be removed from the domain * `-L, --logstore`: If specified, all files in `logstore:` will be removed * `-E, --error-reports`: If specified, all error reports will be removed from the appliance(s) * `-r, --recursive`: If specified, directories will be cleaned recursively * `--no-backup-files`: If specified, files will not be backed up before deleting * `-o, --out-dir`: The directory to save backed up files * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) t = Timestamp() dirs = [] if checkpoints: dirs.append('chkpoints:/') if export: dirs.append('export:/') if logtemp: dirs.append('logtemp:/') if logstore: dirs.append('logstore:/') if web: rows = [] for appliance in env.appliances: if web: rows.append((appliance.hostname, )) for _dir in dirs: _clean_dir(appliance, _dir, Domain, recursive, backup_files, t.timestamp, out_dir) if web: rows.append(("", _dir, "Cleaned")) if error_reports: _clean_error_reports(appliance, Domain, backup_files, t.timestamp, out_dir) rows.append(("", "ErrorReports", "Cleaned")) return flask.render_template( "results_table.html", header_row=["Appliance", "Location", "Action"], rows=rows), util.render_history(env)
def restore_secure_backup(appliances=[], credentials=[], timeout=1200, no_check_hostname=False, CryptoCertificate="", location="", validate_only=False, web=False): """Restores a secure backup to the specified appliances. Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-C, --CryptoCertificate`: The CryptoCertificate object with which the secure backup was encrypted * `-l, --location`: The location on the appliances where the SecureBackup resides (This means that you will have to upload the secure backup if you got it from MAST, external to the appliance) * `-v, --validate-only`: If specified then the appliances will only attemp to validate the backup instead of actually restoring it * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" logger = make_logger("mast.backups") check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) logger.info("Attempting to restore Secure Backup on {}".format( str(env.appliances))) validate = "on" if validate_only else "off" kwargs = { "cred": CryptoCertificate, "source": location, "validate": validate } resp = env.perform_action("SecureRestore", **kwargs) logger.debug("Responses received: {}".format(str(resp))) if web: return (util.render_boolean_results_table(resp), util.render_history(env)) for host, msg in resp.items(): print host, '\n', "=" * len(host) print msg print
def get_secure_backup(appliances=[], credentials=[], timeout=1200, no_check_hostname=False, out_dir='tmp', CryptoCertificate="", destination='local:/raid0', include_iscsi=False, include_raid=False, remove=True, quiesce_before=True, unquiesce_after=True, quiesce_timeout=60, web=False): """Performs a secure backup of the specified domain. Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-o, --out-dir`: (NOT NEEDED IN WEB GUI) The directory (local) to store the backup * `-C, --CryptoCertificate`: The CryptoCertificate object to use to encrypt the backup * `-d, --destination`: The base location (on the appliance) to store the backup * `-i, --include-iscsi`: Whether to include the iscsi filesystem * `-I, --include-raid`: Whether to include the RAID filesystem * `-N, --no-remove`: If specified the backup will NOT be removed from the DataPower * `--no-quiesce-before`: If specified, the appliance will not be quiesced before performing the secure backup * `--no-unquiesce-after`: If specified, the appliance will not be unquiesced after performing the secure backup * `-q, --quiesce-timeout`: The timeout to wait before the appliance attempts to quiesce * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" logger = make_logger("mast.backups") check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) output = "" if quiesce_before: resp = {} for appliance in env.appliances: logger.info("Quiescing {} in preparation of Secure Backup".format( appliance.hostname)) resp[appliance.hostname] = appliance.QuiesceDP( timeout=quiesce_timeout) logger.debug("Response received {}".format( resp[appliance.hostname])) if web: output += util.render_boolean_results_table( resp, suffix="Quiesce_appliance") sleep(quiesce_timeout) t = Timestamp() if destination.endswith("/"): destination = destination.rstrip("/") destination = '%s/%s' % (destination, t.timestamp) kwargs = {'Dir': destination, 'domain': 'default'} logger.info("Creating directory {} on {} to store Secure Backup".format( destination, str(env.appliances))) resp = env.perform_async_action('CreateDir', **kwargs) logger.debug("Responses received {}".format(str(resp))) if web: output += util.render_boolean_results_table(resp, suffix="CreateDir") include_raid = 'on' if include_raid else 'off' include_iscsi = 'on' if include_iscsi else 'off' kwargs = { 'cert': CryptoCertificate, 'destination': destination, 'include_iscsi': include_iscsi, 'include_raid': include_raid } logger.info("Attempting to perform a Secure Backup on {}".format( str(env.appliances))) resp = env.perform_async_action('SecureBackup', **kwargs) logger.debug("Responses received: {}".format(str(resp))) if web: output += util.render_boolean_results_table(resp, suffix="SecureBackup") if web: results = {} remove_results = {} for appliance in env.appliances: directory = os.path.join(out_dir, appliance.hostname, "SecureBackup", t.timestamp) start = time() while not appliance.file_exists( '{}/backupmanifest.xml'.format(destination), 'default'): sleep(5) if time() - start > timeout: raise TimeoutError logger.info("Attempting to retrieve Secure Backup from {}".format( appliance.hostname)) appliance.copy_directory(destination, directory) _directory = os.path.join( directory, destination.replace(":", "").replace("///", "/")) try: logger.info("Attempting to verify Secure Backup for {}".format( appliance.hostname)) if appliance.verify_local_backup(_directory): logger.info("Secure Backup integrity verified for {}".format( appliance.hostname)) if web: results[appliance.hostname] = "Succeeded" else: print '\t', appliance.hostname, " - ", "Succeeded" if remove: logger.info( "Attempting to remove Secure Backup from appliance " "{}".format(appliance.hostname)) _resp = appliance.RemoveDir(Dir=destination, domain='default') logger.debug("Response received: {}".format(_resp)) if web: remove_results[appliance.hostname] = _resp else: logger.warn("Secure Backup for {} Corrupt!".format( appliance.hostname)) if web: results[appliance.hostname] = "Failed" else: print '\t', appliance.hostname, " - ", "Failed" appliance.log_error('Verification of backup in %s failed' % (_directory)) except: if web: results[appliance.hostname] = "Failed" logger.exception( "An unhandled exception occurred during execution.") if web: output += util.render_results_table(results, suffix="verify-SecureBackup") output += util.render_boolean_results_table(remove_results, suffix="RemoveDir") if unquiesce_after: resp = {} for appliance in env.appliances: logger.info("Attempting to unquiesce {}".format( str(appliance.hostname))) resp[appliance.hostname] = appliance.UnquiesceDP() logger.debug("Response received: {}".format( resp[appliance.hostname])) if web: output += util.render_boolean_results_table( resp, suffix="Unquiesce_appliance") if web: return output, util.render_history(env)
def get_normal_backup(appliances=[], credentials=[], timeout=120, no_check_hostname=False, Domain=[], comment="", out_dir='tmp', individual=False, web=False): """Performs a normal backup of the specified domain. Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-D, --Domain`: The domains to backup (all-domains will backup all domains) To spcify multiple domains, use multiple entries of the form `[-D domain1 [-D domain2...]]` * `-C, --comment`: The comment to add to the backup * `-o, --out-dir`: (NOT NEEDED IN WEB GUI) The directory (local) where you would like to store the backup * `-I, --individual`: If specified and all-domains is specified as --Domain then backup each domain individually instead of "all-domains" * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" logger = make_logger("mast.backups") t = Timestamp() check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) if not Domain: raise ValueError( "Must provide one or more domains including 'all-domains'") if isinstance(Domain, basestring): Domain = [Domain] # Fixes duplicate domains issue Domain = list(set(Domain)) results = {} if not individual: logger.info("Attempting to retrieve normal backup from " "{} in {} domain".format(str(env.appliances), Domain)) kwargs = {'domains': Domain, 'comment': comment} _results = env.perform_async_action('get_normal_backup', **kwargs) logger.debug("backups retrieved, check file for contents") for hostname, backup in _results.items(): directory = os.path.join(out_dir, hostname, "NormalBackup", t.timestamp) os.makedirs(directory) filename = os.path.join( directory, '%s-%s-%s.zip' % (t.timestamp, hostname, "_".join(Domain))) logger.debug("Writing backup for {} to {}".format( hostname, filename)) with open(filename, 'wb') as fout: fout.write(backup) if _verify_zip(filename): logger.info("backup for {} in {} domain verified".format( hostname, str(Domain))) results[hostname + "-" + "_".join(Domain) + "-normalBackup"] = "Verified" else: logger.info("backup for {} in {} domain corrupt".format( hostname, str(Domain))) results[hostname + "-" + "_".join(Domain) + "-normalBackup"] = "Corrupt" else: for domain in Domain: logger.info("Attempting to retrieve normal backup from " "{} in {} domain".format(str(env.appliances), domain)) kwargs = {'domains': domain, 'comment': comment} _results = env.perform_async_action('get_normal_backup', **kwargs) logger.debug("backups retrieved, check file for contents") for hostname, backup in _results.items(): directory = os.path.join(out_dir, hostname, "NormalBackup", t.timestamp) if not os.path.exists(directory): os.makedirs(directory) filename = os.path.join( directory, '%s-%s-%s.zip' % (t.timestamp, hostname, domain)) logger.debug("Writing backup for {} to {}".format( hostname, filename)) with open(filename, 'wb') as fout: fout.write(backup) if _verify_zip(filename): logger.info("backup for {} in {} domain verified".format( hostname, domain)) results[hostname + "-" + domain + "-normalBackup"] = "Verified" else: logger.info("backup for {} in {} domain corrupt".format( hostname, domain)) results[hostname + "-" + domain + "-normalBackup"] = "Corrupt" if web: return util.render_results_table(results), util.render_history(env) for k, v in results.items(): print print k print '=' * len(k) print v print
def predeploy(appliances=[], credentials=[], timeout=120, no_check_hostname=False, out_dir="tmp", Domain="", comment="", predeploy_command=None, CryptoCertificate="", secure_backup_destination="local:/raid0", backup_default=True, backup_all=True, do_secure_backup=False, do_normal_backup=True, set_checkpoints=True, include_iscsi=False, include_raid=False, remove_secure_backup=True, default_checkpoint=True, remove_oldest_checkpoint=True, allow_shell=False, web=False): """Perform routine pre-deployment actions. Everything is optional, but if you wish to perform an action, you must provide the necessary arguments. Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-o, --out-dir`: (Not needed in web GUI) The directory (local) where you would like to store the artifacts generated by this script * `-D, --Domain`: This is the app domain to backup and the app domain in which to set a checkpoint * `-C, --comment`: This is shared among other actions. The comment is used to build the name of the checkpoint. * `-p, --predeploy-command`: This command will be "shelled out" to the machine running MAST after performing the backups and checkpoints. Use this parameter to pull from version control or similar operations. * `--CryptoCertificate`: The CryptoCertificate with which to encrypt the secure backup * `-s, --secure-backup-destination`: The destination (on the DataPower) where the secure backup will be stored * `-N, --no-backup-default`: Whether to also backup the default domain * `--no-backup-all`: Whether to also backup all-domains * `-d, --do-secure-backup`: Whether to retrieve a secure backup * `--no-do-normal-backup`: Whether to retrieve normal backups * `--no-set-checkpoints`: Whether to set checkpoints * `-i, --include-iscsi`: Whether to include the iscsi volume in the secure backup * `-I, --include-raid`: Whether to include the raid volume in the secure backup * `--no-remove-secure-backup`: Whether to remove the secure backup from the appliance after verifying your local copy. * `--no-default-checkpoint`: Whether to create a checkpoint in the default domain * `--no-remove-oldest-checkpoint`: Whether to remove the oldest checkpoint from the domain IF AND ONLY IF the maximum number of checkpoints has been reached. * `-A, --allow-shell`: Whether to allocate a shell for the execution of the predeploy-command * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__ Here is what is possible (will be done in this order): 1. Secure Backup * The following parameters apply: * `-d, --do-secure-backup` * `-o, --out-dir` * `--CryptoCertificate` * `-s, --secure-backup-destination` * `-i, --include-iscsi` * `-I, --include-raid` * `--no-remove-secure-backup` 2. Normal Backups * The following parameters apply: * `--no-do-normal-backup` * `-o, --out-dir` * `-D, --Domain` * `-N, --no-backup-default` * `--no-backup-all` * `-C, --comment` 3. Checkpoints * The following parameters apply: * `--no-set-checkpoints` * `-D, --Domain` * `-C, --comment` * `--no-default-checkpoint` * `--no-remove-oldest-checkpoint`""" from mast.datapower.backups import set_checkpoint from mast.datapower.backups import get_normal_backup, get_secure_backup logger = make_logger('mast.datapower.deployment') check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) if web: output = "" history = "" # Loop through appliances so we will only affect one appliance at a time for appliance in env.appliances: # Secure Backup if do_secure_backup: if not CryptoCertificate: # Fail if CryptoCertificate is not given logger.error("Cert must be specified in order " "to perform a secure backup!") sys.exit(-1) logger.info("Starting Secure Backup for {}".format( appliance.hostname)) _out = get_secure_backup(appliances=appliance.hostname, credentials=appliance.credentials, timeout=timeout, out_dir=out_dir, CryptoCertificate=CryptoCertificate, destination=secure_backup_destination, include_iscsi=include_iscsi, include_raid=include_raid, remove=remove_secure_backup, quiesce_before=False, unquiesce_after=False, no_check_hostname=no_check_hostname, web=web) if web: output += _out[0] history += _out[1] logger.info("Finished Secure Backup for {}".format( appliance.hostname)) # Normal backups if do_normal_backup: logger.info("Pre-Deployment backups started at {}".format( str(Timestamp()))) domains = [Domain] if backup_default: domains.append("default") if backup_all: domains.append("all-domains") _out = get_normal_backup(appliances=[appliance.hostname], credentials=credentials, timeout=timeout, no_check_hostname=no_check_hostname, Domain=domains, comment=comment, out_dir=out_dir, web=web) logger.info("Pre-Deployment backups finished at {}".format( str(Timestamp()))) if web: output += _out[0] history += _out[1] # Checkpoints if set_checkpoints: logger.info("Pre-Deployment checkpoints started at {}".format( str(Timestamp()))) domains = [Domain] if default_checkpoint: domains.append("default") _out = set_checkpoint(appliances=[appliance.hostname], credentials=credentials, timeout=timeout, no_check_hostname=no_check_hostname, Domain=domains, comment=comment, remove_oldest=remove_oldest_checkpoint, web=web) logger.info("Pre-Deployment checkpoints finished at {}".format( str(Timestamp()))) if web: output += _out[0] history += _out[1] if predeploy_command: logger.info( "Pre-Deployment command '{}' found. Executing at {}".format( predeploy_command, str(Timestamp()))) out, err = system_call(command=predeploy_command, shell=allow_shell) out = str(out) err = str(err) logger.info( "finished executing Pre-Deployment command '{}' at {}., output: {}" .format(predeploy_command, str(Timestamp()), ";".join([out, err]))) if web: from mast.plugin_utils.plugin_utils import render_results_table results = { "predeploy command": "{}\n\nout: {}\n\nerr: {}".format(predeploy_command, out, err) } output += render_results_table(results) else: print "Finished running pre-deploy command. output: {}".format( ";".join([out, err])) if web: return output, history
def postdeploy(appliances=[], credentials=[], timeout=120, no_check_hostname=False, Domain="", unquiesce_domain=True, unquiesce_appliance=False, postdeploy_command=None, save_config=True, web=False): """This is a simple script which will allow you to unquiesce your domain or appliances after you quiesce them for a deployment. Also this will allow you to save the config. Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-D, --Domain`: The domain which will be unquiesced and persisted. * `-N, --no-unquiesce-domain`: If specified, this script will not attempt to unquiesce the domain * `-u, --unquiesce-appliance`: If specified, this script will attempt to unquiesce the appliance * `-p, --postdeploy-command`: This command will be "shelled out" to the machine running MAST after unquiescing and saving the configuration, use this parameter to clean up VCS artifacts and/or perform check-outs of your deployed services or similar operations * `--no-save-config`: If specified, the configuration will not be saved in the application domain * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" import mast.datapower.system as system check_hostname = not no_check_hostname logger = make_logger('mast.datapower.deployment') env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) if web: output = "" history = "" for appliance in env.appliances: if unquiesce_appliance: appliance.log_info("Attempting to unquiesce appliance") _out = system.unquiesce_appliance( appliances=[appliance.hostname], credentials=[appliance.credentials], timeout=timeout, no_check_hostname=no_check_hostname, web=web) appliance.log_info("Finished Unquiescing appliance") if web: output += _out[0] history += _out[1] else: print "Finished unquiescing appliance" if unquiesce_domain: appliance.log_info("Attempting to unquiesce domain") _out = system.unquiesce_domain(appliances=[appliance.hostname], credentials=[appliance.credentials], timeout=timeout, no_check_hostname=no_check_hostname, Domain=Domain, web=web) appliance.log_info("Finished Unquiescing domain") if web: output += _out[0] history += _out[1] else: print "Finished unquiescing domain" if save_config: appliance.log_info( "Attempting to save configuration after deployment") _out = system.save_config(appliances=[appliance.hostname], credentials=[appliance.credentials], timeout=timeout, no_check_hostname=no_check_hostname, Domain=Domain, web=web) appliance.log_info( "Finished saving configuration after deployment") if web: output += _out[0] history += _out[1] else: print "Finished saving the configuration" if postdeploy_command: logger.info( "Post-Deployment command '{}' found. Executing at {}".format( postdeploy_command, str(Timestamp()))) out, err = system_call(command=postdeploy_command) out = str(out) err = str(err) logger.info( "finished executing Post-Deployment command '{}' at {}., output: {}" .format(postdeploy_command, str(Timestamp()), ";".join([out, err]))) if web: from mast.plugin_utils.plugin_utils import render_results_table results = { "postdeploy command": "{}\n\nout: {}\n\nerr: {}".format(postdeploy_command, out, err) } output += render_results_table(results) else: print "Finished running post-deploy command. output: {}".format( ";".join([out, err])) if web: return output, history
def cert_file_audit(appliances=[], credentials=[], timeout=120, no_check_hostname=False, out_file=os.path.join("tmp", "cert-file-audit.xlsx")): locations = ["cert:", "pubcert:", "sharedcert:"] check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) header_row = ["appliance", "domain", "filename", "base-filename", "size", "modified"] rows = [] for appliance in env.appliances: print appliance.hostname domain = "default" for location in locations: print "\t{}".format(location) filestore = appliance.get_filestore(domain=domain, location=location) _location = filestore.xml.find(datapower.FILESTORE_XPATH) if _location is None: continue if _location.findall("./file") is not None: for _file in _location.findall("./file"): dir_name = _location.get("name") filename = _file.get("name") filename = "/".join((dir_name, filename)) filename = re.sub(r":[/]*", ":///", filename) base_filename = filename.split("/")[-1] print "\t\t{}".format(filename) size = _file.find("size").text modified = _file.find("modified").text rows.append([appliance.hostname, domain, filename, base_filename, size, modified]) for directory in _location.findall(".//directory"): dir_name = directory.get("name") print "\t\t{}".format(dir_name) for _file in directory.findall(".//file"): filename = _file.get("name") filename = "/".join((dir_name, filename)) filename = re.sub(r":[/]*", ":///", filename) base_filename = filename.split("/")[-1] # Infer domain from second segment of path if in cert: _domain = domain if location == "cert:": _domain = filename.replace("///", "/").split("/")[1] filename = filename.replace(_domain+"/", "") print "\t\t\t{}".format(filename) size = _file.find("size").text modified = _file.find("modified").text rows.append([appliance.hostname, _domain, filename, base_filename, size, modified]) wb = openpyxl.Workbook() ws = wb.get_active_sheet() ws.title = "CertFileAudit" ws.append(header_row) for row in rows: ws.append(row) wb.save(out_file)
def cert_audit(appliances=[], credentials=[], timeout=120, no_check_hostname=False, domains=[], out_file="tmp/cert-audit.xlsx", delay=0.5): check_hostname = not no_check_hostname env = datapower.Environment( appliances, credentials, timeout, check_hostname=check_hostname) wb = openpyxl.Workbook() ws = wb.get_active_sheet() header_row = [ "appliance", "domain", "certificate-object", "filename", "subject", "SANs", "not-before", "not-after", "issuer"] ws.append(header_row) for appliance in env.appliances: logger.info("Checking appliance {}".format(appliance.hostname)) print appliance.hostname _domains = domains if "all-domains" in domains: _domains = appliance.domains for domain in _domains: logger.info("In domain {}".format(domain)) print "\t", domain config = appliance.get_config("CryptoCertificate", domain=domain) certs = [x for x in config.xml.findall(datapower.CONFIG_XPATH)] # Filter out disabled objects because the results won't change, # but we will perform less network traffic certs = filter( lambda x: x.find("mAdminState").text == "enabled", certs) for cert in certs: logger.info("Exporting cert {}".format(cert)) filename = cert.find("Filename").text filename = re.sub(r":[/]*", ":///", filename) name = cert.get("name") _filename = name print "\t\t", name row = [appliance.hostname, domain, name, filename] appliance.CryptoExport( domain=domain, ObjectType="cert", ObjectName=name, OutputFilename=_filename) logger.info("Finished exporting cert {}".format(cert)) try: logger.info( "Retrieving file {}".format( "temporary:///{}".format(_filename))) cert = appliance.getfile( domain, "temporary:///{}".format(_filename)) logger.info( "Finished retrieving file {}".format( "temporary:///{}".format(_filename))) logger.info( "Attempting to delete file {}".format( "temporary:///{}".format(_filename))) appliance.DeleteFile( domain=domain, File="temporary:///{}".format(_filename)) logger.info( "Finished deleting file {}".format( "temporary:///{}".format(_filename))) except UnboundLocalError: # This most likely means the file did not exist for # some reason (skip) continue except: logger.exception("An unhandled exception has occurred") print appliance.history print "SKIPPING CERT" continue cert = etree.fromstring(cert) _contents = insert_newlines(cert.find("certificate").text) certificate = \ "-----BEGIN CERTIFICATE-----\n" +\ _contents +\ "\n-----END CERTIFICATE-----\n" _cert = OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_PEM, certificate) subject = "'{}'".format( ";".join( ["=".join(x) for x in _cert.get_subject().get_components()])) sans = [] ext_count = _cert.get_extension_count() for i in range(0, ext_count): ext = _cert.get_extension(i) if 'subjectAltName' in str(ext.get_short_name()): sans.append(ext.__str__()) sans = "\n".join(sans) issuer = "'{}'".format( ";".join( ["=".join(x) for x in _cert.get_issuer().get_components()])) row.extend( [subject, sans, _cert.get_notBefore(), _cert.get_notAfter(), issuer]) ws.append(row) sleep(delay) wb.save(out_file)
def main( appliances=[], credentials=[], timeout=120, no_check_hostname=False, domains=[], object_classes=[], persisted=False, recursive=False, out_file="tmp/get-config.xlsx", seperator="\n", obfuscate_passwords=False, ): check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout=timeout, check_hostname=check_hostname) worksheets = OrderedDefaultDict(lambda: [ ["appliance", "domain", "name"], ]) if not object_classes: object_classes = [None] for appliance in env.appliances: print appliance.hostname _domains = domains if "all-domains" in _domains: _domains = appliance.domains for domain in _domains: print "\t{}".format(domain) for object_class in object_classes: config = etree.fromstring( str( appliance.get_config(domain=domain, _class=object_class, recursive=recursive, persisted=persisted))) object_status = etree.fromstring( str(appliance.get_status("ObjectStatus", domain=domain))) # First sheet should be ServicesStatusPlus services_status = etree.fromstring( str( appliance.get_status("ServicesStatusPlus", domain=domain))) sheet = worksheets["ServicesStatusPlus"] sheet[0] = [ "appliance", "domain", "ServiceName", "ServiceClass", "LocalIP", "LocalPort", "FshClass", "FshName", "FshStatus", "GatewayStatus", "Summary", "BackendUrl", "BackendHostName", "BackendPort", "FshDirectory", "FshGetQ", "FshTopic", "FshTopicSelection", "FshRemoteServer", "FshRemotePort", "EnableHTTP", "HTTPPort", "EnableHTTPS", "HTTPSPort", "PrimaryInterface", "JunctionType", "JunctionTypeStandard", "JunctionTypeVirtual", "UnresolvedLocalAddress", ] for node in services_status.findall(datapower.STATUS_XPATH): sheet.append([ appliance.hostname, domain, [{ "hostname": appliance.hostname, "domain": domain, "class": node.find("ServiceClass").text, "name": node.find("ServiceName").text, }], node.find("ServiceClass").text, node.find("LocalIP").text, node.find("LocalPort").text, node.find("FshClass").text, node.find("FshName").text, node.find("FshStatus").text, node.find("GatewayStatus").text, node.find("Summary").text, node.find("BackendUrl").text, node.find("BackendHostName").text, node.find("BackendPort").text, node.find("FshDirectory").text, node.find("FshGetQ").text, node.find("FshTopic").text, node.find("FshTopicSelection").text, node.find("FshRemoteServer").text, node.find("FshRemotePort").text, node.find("EnableHTTP").text, node.find("HTTPPort").text, node.find("EnableHTTPS").text, node.find("HTTPSPort").text, node.find("PrimaryInterface").text, node.find("JunctionType").text, node.find("JunctionTypeStandard").text, node.find("JunctionTypeVirtual").text, node.find("UnresolvedLocalAddress").text, ]) for node in config.findall(datapower.CONFIG_XPATH): object_class = node.tag name = node.get("name") print "\t\t{} - {}".format(object_class, name) append_row(worksheets[object_class], appliance.hostname, domain, node, object_status, seperator=seperator, obfuscate_passwords=obfuscate_passwords) if out_file.endswith(".xlsx"): workbook = openpyxl.Workbook() workbook.remove_sheet(workbook.active) for title, data in worksheets.items(): if len(title) > 31: title = "...".join((title[:14], title[-14:])) worksheet = workbook.create_sheet(title=title) for row in data: for index, cell in enumerate(list(row)): if isinstance(cell, list): row[index] = seperator.join( (item["name"] for item in cell)) worksheet.append(row) workbook.save(out_file) elif out_file.endswith(".sqlite"): con = sqlite3.connect(out_file) for name, data in worksheets.items(): for row_index, row in enumerate(list(data)): for cell_index, cell in enumerate(list(row)): if isinstance(cell, list): data[row_index][cell_index] = seperator.join( (item["name"] for item in cell)) df = pd.DataFrame(data, columns=data.pop(0)) df.to_sql(name=name, con=con, if_exists="replace", index=0) elif out_file.endswith(".html"): html = """<!DOCTYPE html> <html> <head> <meta charset="utf-8" /> <meta http-equiv="X-UA-Compatible" content="IE=edge" /> <meta name="viewport" content="width=device-width, initial-scale=1" /> <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" /> <link rel="stylesheet" href="https://cdn.datatables.net/1.10.19/css/jquery.dataTables.min.css" /> </head> <body> <style> tfoot { display: table-header-group; } </style> <!-- <div class="container mx-auto"> --> <ul id="tabs" class="nav nav-tabs" role="tablist" style="display: none;"> """ for title in sorted(worksheets.keys()): html += """<li class="nav-item"> <a class="nav-link" role="tab" data-toggle="tab" href="#{0}">{0}</a> </li> """.format(title) html += "</ul>" html += """<form><select id="tab-select">""" for index, title in enumerate(sorted(worksheets.keys())): if title == "ServicesStatusPlus": html += """<option value="{}" selected="selected">{}</option>""".format( index, title) else: html += """<option value="{}">{}</option>""".format( index, title) html += "</select></form>" html += """<div class="tab-content" id="nav-tabContent">""" for index, title in enumerate(sorted(worksheets.keys())): data = worksheets[title] if title == "ServicesStatusPlus": html += """<div class="tab-pane fade show active" id="{0}" role="tabpanel"><table class="display table table-striped">""".format( title) else: html += """<div class="tab-pane fade" id="{0}" role="tabpanel"><table class="display table table-striped">""".format( title) header = data.pop(0) html += "<thead><tr>" for cell in header: html += """<th>{}</th>""".format(cell) html += "</tr></thead>" html += "<tfoot><tr>" for cell in header: html += """<th></th>""" html += "</tr></tfoot><tbody>" for row in data: if isinstance(row[2], list): html += """<tr id="{}-{}-{}-{}">""".format( row[0].replace(" ", "-"), row[1].replace(" ", "-"), title.replace(" ", "-"), row[2][0]["name"].replace(" ", "-")) else: html += """<tr id="{}-{}-{}-{}">""".format( row[0].replace(" ", "-"), row[1].replace(" ", "-"), title.replace(" ", "-"), row[2].replace(" ", "-")) for cell in row: if isinstance(cell, list): html += "<td>" for item in cell: link = "#{}-{}-{}-{}".format( item["hostname"].replace(" ", "-"), item["domain"].replace(" ", "-"), item["class"].replace(" ", "-"), item["name"].replace(" ", "-"), ) html += """<a href="{}" class="link" data-class="{}" >{}</a><br/>""".format( link, item["class"], item["name"]) html += "</td>" else: if cell: html += "<td>{}</td>".format( cell.replace(seperator, "<br />")) else: html += "<td>null</td>" if len(row) < len(header): for i in range(len(header) - len(row)): html += "<td>null</td>" html += "</tr>" html += "</tbody></table></div>" html += """ <!-- </div> --> </div> <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"></script> <!-- <script src="https://cdnjs.cloudflare.com/ajax/libs/tether/1.4.0/js/tether.min.js"></script> --> <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js"></script> <script src="https://cdn.datatables.net/1.10.19/js/jquery.dataTables.min.js"></script> <script> $(document).ready(function() { $('#tab-select').on('change', function (e) { $('#tabs li a').eq($(this).val()).tab('show'); }); $(".link").on("click", function(e){ e.preventDefault(); var objectClass = $(e.target).attr("data-class"); $(".nav-tabs a[href='#" + objectClass + "']").one("shown.bs.tab", function(){ $($(e.target).attr("href")).css("background-color", "yellow"); $($(e.target).attr("href")).get(0).scrollIntoView(); $('body, html').scrollLeft(0); }); $("#tab-select").val($("option:contains('" + objectClass + "')").filter(function(){return $(this).text() === objectClass}).val()); $("#tab-select").trigger( "change" ); }); var tables = $('table').DataTable({ paging: false }); $.each($('table'), function () { $(this).find('tfoot th').each(function (i) { var currentTable = $(this).closest('table').DataTable(); var select = $('<input type="text">') .appendTo($(this)) .on('change', function () { var val = $(this).val(); console.log(val ? $(this).val() : val) currentTable.column(i) .search(val ? $(this).val() : val, false, true) .draw(); }); }); }) } ); </script> </body> </html>""" with open(out_file, "w") as fp: fp.write(html) else: raise NotImplementedError( "Unsopported output type, choose one of ['.xlsx', '.sqlite3', '.html']" )
def list_probes(appliances=[], credentials=[], timeout=120, no_check_hostname=False, Domain=[], web=False): """Lists all enabled probes in all specified domains Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-D, --Domain`: One or more domains to inspect. To spcify multiple domains, use multiple entries of the form `[-D domain1 [-D domain2...]]` * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" import urllib2 check_hostname = not no_check_hostname env = datapower.Environment( appliances, credentials, timeout=timeout, check_hostname=check_hostname) results = {} for appliance in env.appliances: domains = Domain if "all-domains" in Domain: domains = appliance.domains for domain in domains: try: config = appliance.get_config( _class="all-classes", name="all-objects", domain=domain, persisted=False) except urllib2.HTTPError: config = appliance.get_config(domain=domain, persisted=False) for obj in config.xml.findall(datapower.CONFIG_XPATH): if obj.find("DebugMode") is not None: if obj.find("DebugMode").text == "on": k = "{}-{}".format(appliance.hostname, domain) v = "{} - {}".format(obj.tag, obj.get("name")) if k in results: results[k].append(v) else: results[k] = [v] if web: for k, v in results.items(): results[k] = "\n".join(v) return ( util.render_results_table(results), util.render_history(env)) else: for k, v in results.items(): print k, "\n", "-" * len(k) for item in v: print "\t{}".format(item)
def export(appliances=[], credentials=[], timeout=120, no_check_hostname=False, Domain="", object_name=None, object_class=None, comment='', format='ZIP', persisted=True, all_files=True, referenced_files=True, referenced_objects=True, out_dir='tmp', web=False): """Exports a service or object to be used to import into another domain or appliance Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-D, --Domain`: The domain from which to export service/object * `-o, --object-name`: The name of the object to export * `-O, --object-class`: The class of the object to export * `-C, --comment`: The comment to embed into the export * `-f, --format`: the format in which to export the configuration. This can be either "XML" or "ZIP" * `-N, --no-persisted`: If specified, the running configuration will be exported as opposed to the persisted configuration * `--no-all-files`: If specified, the export will not include all files * `--no-referenced-files`: If specified, the referenced files will not be included in the export * `--no-referenced-objects`: If specified, referenced objects will not be included in the export. * `--out-dir`: (**NOT NEEDED IN THE WEB GUI**)The directory (local) in which to save the export * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" logger = make_logger("mast.developer") t = Timestamp() if object_name is None or object_class is None: try: raise TypeError("Must Provide both object name and object class") except: logger.exception("Must Provide both object name and object class") raise check_hostname = not no_check_hostname env = datapower.Environment( appliances, credentials, timeout, check_hostname=check_hostname) msg = "Attempting to export {} from {}".format(object_name, str(env.appliances)) logger.info(msg) kwargs = { 'domain': Domain, 'obj': object_name, 'object_class': object_class, 'comment': comment, 'format': format, 'persisted': persisted, 'all_files': all_files, 'referenced_objects': referenced_objects, 'referenced_files': referenced_files} results = env.perform_action( 'export', **kwargs) for hostname, _export in results.items(): d = os.path.join(out_dir, hostname, t.timestamp) os.makedirs(d) extention = format.lower() filename = os.path.join(d, '%s-%s-%s.%s' % ( t.timestamp, hostname, object_name, extention)) msg = "Writing export of {} from {} to {}".format(object_name, hostname, filename) logger.debug(msg) if not web: print msg with open(filename, 'wb') as fout: fout.write(_export) if web: return util.render_see_download_table( results, suffix="export"), util.render_history(env)
def deploy(appliances=[], credentials=[], timeout=180, no_check_hostname=False, Domain=[], file_in=None, deployment_policy="", dry_run=False, overwrite_files=True, overwrite_objects=True, rewrite_local_ip=True, object_audit=True, out_dir='tmp', format='ZIP', quiesce_domain=True, quiesce_appliance=False, quiesce_timeout=120, web=False): """Perform a deployment/migration of a service/object to an IBM DataPower appliance. This script will try to perform the deployment/migration in a manner consistent with best practices. __WARNING__: There is an inherent security risk involved in this script, in order to allow the most flexible integration with various Version Control Systems possible, we allow a pre-deployment hook and a post-deployment hook which will be "shelled out" to your operating system. For this reason PLEASE be sure to run this script (and the MAST Web GUI server) as a user with appropriate permissions. DO NOT RUN AS ROOT!!! Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-D, --Domain`: The domain to import the configuration into, also the domain to quiesce if `--no-quiesce-domain` is not specified. * `-f, --file-in`: This is the configuration file that you are importing. It must be in the format specified by the format parameter. * `-d, --deployment-policy`: The deployment policy to apply to the import, must already exist on the appliance * `--dry-run`: If specified, a dry-run will be performed instead of an actual import * `-N, --no-overwrite-files`: If specified, no files will be over- written during the import * `--no-overwrite-objects`: If specified, no objects will be over- written during import * `--no-rewrite-local-ip`: If specified local ip addresses will not be rewritten on import * `--no-object-audit`: If specified, an object audit will not be performed. An object audit is a diff between the running and persisted configuration * `-o, --out-dir`: This is where to place the artifacts generated by this script * `-F, --format`: The format of the configuration file, must be either "ZIP" or "XML". * `--no-quiesce-domain`: If specified, the domain will not be quiesced prior to the deployment. * `-q, --quiesce-appliance`: If specified, the appliance will be quiesced prior to the deployment * `-Q, --quiesce-timeout`: This is the amount of time for the appliance to wait before beginning the quiescence procedure. * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" import mast.datapower.system as system from mast.datapower.developer import _import logger = make_logger('mast.datapower.deployment') if web: output = "" history = "" check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) for appliance in env.appliances: appliance.log_info("Deployment started on {}".format( appliance.hostname)) # Quiesce Domain if quiesce_domain: logger.info("Quiescing domain {} before deployment at {}".format( Domain, str(Timestamp()))) _out = system.quiesce_domain(appliances=[appliance.hostname], credentials=credentials, timeout=timeout, no_check_hostname=no_check_hostname, Domain=Domain, quiesce_timeout=quiesce_timeout, web=web) logger.info( "Finished quiescing domain {} before deployment at {}".format( Domain, str(Timestamp()))) sleep(quiesce_timeout) if web: output += _out[0] history += _out[1] # Quiesce Appliance if quiesce_appliance: logger.info("Quiescing appliances before deployment at {}".format( str(Timestamp()))) _out = system.quiesce_appliance( appliances=appliance.hostname, credentials=credentials, timeout=timeout, no_check_hostname=no_check_hostname, quiesce_timeout=quiesce_timeout, web=web) logger.info( "Finished quiescing appliances before deployment at {}".format( str(Timestamp()))) sleep(quiesce_timeout) if web: output += _out[0] history += _out[1] appliance.log_info("Attempting to import configuration at '{}'".format( str(Timestamp()))) file_out = os.path.join( out_dir, '{}-deployment_results.txt'.format(appliance.hostname)) # import configuration _out = _import(appliances=[appliance.hostname], credentials=credentials, timeout=timeout, no_check_hostname=no_check_hostname, Domain=Domain, file_in=file_in, deployment_policy=deployment_policy, dry_run=dry_run, overwrite_files=overwrite_files, overwrite_objects=overwrite_objects, rewrite_local_ip=rewrite_local_ip, source_type=format, out_dir=out_dir, web=web) if web: output += _out[0] history += _out[1] appliance.log_info("Finished importing configuration at {}".format( str(Timestamp()))) # unquiesce domain if quiesce_domain: appliance.log_info("Attempting to unquiesce domain") _out = system.unquiesce_domain(appliances=[appliance.hostname], credentials=credentials, timeout=timeout, no_check_hostname=no_check_hostname, Domain=Domain, web=web) appliance.log_info("Finished unquiescing domain") if web: output += _out[0] history += _out[1] # unquiesce appliance if quiesce_appliance: logger.info("Quiescing appliances before deployment at {}".format( str(Timestamp()))) _out = system.unquiesce_appliance( appliances=[appliance.hostname], credentials=credentials, timeout=timeout, no_check_hostname=no_check_hostname, web=web) logger.info( "Finished quiescing appliances before deployment at {}".format( str(Timestamp()))) if web: output += _out[0] history += _out[1] if object_audit: appliance.log_info( "Post-Deployment Object audit started at {}".format( str(Timestamp()))) _out = system.objects_audit(appliances=[appliance.hostname], credentials=credentials, timeout=timeout, no_check_hostname=no_check_hostname, out_dir=out_dir, web=web) appliance.log_info( "Post-Deployment Object audit finished at {}".format( str(Timestamp()))) if web: output += _out[0] history += _out[1] if web: return output, history
def main(appliances=[], credentials=[], timeout=120, no_check_hostname=False, base_dir=default_base_dir, comment=default_comment, persisted=False, recursive=False, no_strip_timestamp=False, page=False): """ track_getconfig.py Description: Store running or persisted domain configuration in a local git repository for auditing purposes. Usage: :::bash $ mast contrib/track_getconfig.py --appliances <HOSTNAMES> --credentials <USER:PASS> --base-dir tmp/config Parameters: * `-a, --appliances` - The hostname(s), ip addresse(s), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password` and should be provided in a space-seperated list if multiple are provided. If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-b, --base-dir`: The base directory where to store the downloaded files. Files will actually be stored in a subdirectory of `base_dir` named after the hostname in the form of `base_dir/<hostname>` * `-p, --persisted`: If specified, the persisted configuration will be retrieved as opposed to the running configuration (which is the default) * `-r, --recursive`: If specified, the configuration will be retrieved recursively to the extent of the facilities provided by DataPower * `-N, --no-strip-timestamp`: If specified, the timestamp will not be stripped from the XML document, This is done because we are tracking this information with git and stripping this out allows us to alert on any changes in the repository as opposed to special-casing the difference in timestamp. * `-P, --page`: If specified, page the output when too long to display at once """ base_dir = os.path.abspath(base_dir) if not os.path.exists(base_dir): os.makedirs(base_dir) try: repo = git.Repo(base_dir) repo.head() except NotGitRepository: print "Initializing git repository" git.init(base_dir) git.add(base_dir) git.commit(base_dir, message="Initial Commit") except KeyError: git.add(base_dir) git.commit(base_dir, message="Initial Commit") check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout=timeout, check_hostname=check_hostname) for appliance in env.appliances: appliance_directory = os.path.join(base_dir, appliance.hostname) if not os.path.exists(appliance_directory): os.mkdir(appliance_directory) for domain in appliance.domains: config = appliance.get_config(domain=domain, recursive=recursive, persisted=persisted) config = config.pretty if no_strip_timestamp: pass else: config = re.sub(r"^.*?<dp:timestamp>.*?</dp:timestamp>.*?$", r"", config, flags=re.MULTILINE) filename = os.path.join(appliance_directory, "{}.xml".format(domain)) with open(filename, "wb") as fout: fout.write(config) git.add(base_dir) print git.status(base_dir) git.commit(base_dir, message=comment) tmp = StringIO() git.show(base_dir, outstream=tmp) tmp.seek(0) out = highlight(tmp.read(), DiffLexer(), TerminalFormatter()) if page: pprint.page(out) else: print out
def set_checkpoint(appliances=[], credentials=[], timeout=120, no_check_hostname=False, Domain=['default'], comment='', remove_oldest=True, web=False): """Sets a checkpoint in the given domains on the specified appliances Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-D, --Domain`: Domains to set checkpoints in. To spcify multiple domains, use multiple entries of the form `[-D domain1 [-D domain2...]]` * `-C, --comment`: The comment to use for the checkpoint (will also be used to build the checkpoint name) * `-N, --no-remove-oldest`: If specified this script will attempt to remove the oldest checkpoint __IF__ the maximum number of checkpoints exist * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" logger = make_logger("mast.backups") check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) logger.info("Attempting to set checkpoint on {} in {} domain(s)".format( str(env.appliances), str(Domain))) t = Timestamp() if web: header_row = ("Appliance", "Result") rows = [] for appliance in env.appliances: if not web: print appliance.hostname _domains = Domain print Domain print _domains if "all-domains" in _domains: _domains = appliance.domains print _domains for domain in _domains: print domain if not web: print "\t", domain name = '{0}-{1}-{2}'.format(comment, domain, t.timestamp) logger.debug( "Attempting to set checkpoint {} on {} in {} domain".format( name, appliance, domain)) if remove_oldest: _max = appliance.max_checkpoints(domain) if len(appliance.get_existing_checkpoints(domain)) >= _max: logger.info( "Maximum number of checkpoints for domain " "{} on {} reached. Removing oldest checkpoint.".format( domain, appliance.hostname)) _resp = appliance.remove_oldest_checkpoint(domain) logger.debug("Response received: {}".format(_resp)) kwargs = {'domain': domain, 'ChkName': name} resp = appliance.SaveCheckpoint(**kwargs) logger.debug("Response received: {}".format(resp)) if not web: if resp: print "\t\tSuccessful" else: print "\t\tFailed" if web: if resp: rows.append( ("{}-{}-set_checkpoint".format(appliance.hostname, domain), "Succeeded")) else: rows.append( ("{}-{}-set_checkpoint".format(appliance.hostname, domain), "Failed")) if web: return flask.render_template("results_table.html", header_row=header_row, rows=rows), util.render_history(env)
def _import(appliances=[], credentials=[], timeout=120, no_check_hostname=False, Domain=[], file_in=None, deployment_policy=None, deployment_policy_variables=None, dry_run=False, overwrite_files=True, overwrite_objects=True, rewrite_local_ip=True, source_type='ZIP', out_dir="tmp/", web=False): """Import a service/object into the specified appliances Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-D. --Domain`: The domain into which the configuration will be imported * `-f, --file-in`: The file to import into the specified domain. This __MUST__ match the format specified in source_type * `-d, --deployment-policy`: The deployment policy to use for the import (must already exist on the appliances) * `--dry-run`: Whether to do a dry-run (nothing will be imported) * `-N, --no-overwrite-files`: If specified, no files will be overwritten as part of the import * `--no-overwrite-objects`: If specified, no objects will be overwritten as part of the import * `--no-rewrite-local-ip`: If specified, no local ip addresses will be rewritten as part of the import * `-s, --source-type`: The type of file to import. Can be "XML" or "ZIP" * `-o, --out-dir`: The directory to output artifacts generated by this script * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" logger = make_logger("mast.developer") t = Timestamp() check_hostname = not no_check_hostname env = datapower.Environment( appliances, credentials, timeout, check_hostname=check_hostname) msg = "Attempting to import {} to {}".format(file_in, str(env.appliances)) logger.info(msg) if not web: print msg results = {} out_dir = os.path.join(out_dir, "import_results", t.timestamp) os.makedirs(out_dir) for appliance in env.appliances: if not web: print appliance.hostname results[appliance.hostname] = {} domains = Domain if "all-domains" in domains: domains = appliance.domains for domain in domains: if not web: print "\t", domain kwargs = { 'domain': domain, 'zip_file': file_in, 'deployment_policy': deployment_policy, 'deployment_policy_variables': deployment_policy_variables, 'dry_run': dry_run, 'overwrite_files': overwrite_files, 'overwrite_objects': overwrite_objects, 'rewrite_local_ip': rewrite_local_ip, 'source_type': source_type} resp = appliance.do_import(**kwargs) results[appliance.hostname][domain] = resp if not web: pprint_xml(resp.xml) logger.debug("Response received: {}".format(str(resp))) filename = os.path.join( out_dir, "{}-{}-import_results.xml".format( appliance.hostname, domain ) ) with open(filename, 'wb') as fout: fout.write(resp.pretty) if web: return util.render_see_download_table( results, suffix="import"), util.render_history(env)
def main(appliances=[], credentials=[], timeout=120, EthernetInterface="", add=[], remove=[], save_config=False): """ _Script_: `contrib/ssh-add-static-route.py` DESCRIPTION: Adds or removes multiple static routes to an EthernetInterface. USAGE: To add two static routes and remove two static routes from two appliances, this is the form you should use: ``` $ mast contrib/ssh-routes.py --appliances APPL_1 --appliances APPL_2 --credentials user:pass --EthernetInterface INT_NAME --add "xxx.xxx.xxx.xxx/xx xxx.xxx.xxx.xxx x" --add "xxx.xxx.xxx.xxx/xx xxx.xxx.xxx.xxx x" --remove "xxx.xxx.xxx.xxx/xx xxx.xxx.xxx.xxx x" --remove "xxx.xxx.xxx.xxx/xx xxx.xxx.xxx.xxx x" ``` RETURNS PARAMETERS: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-E, --EthernetInterface`: The name of the EthernetInterface to which to add the static route. * `-A, --add`: The static route(s) to add, should be in the form of "ip/cidr gateway metric" Where ip/cidr is the destination IP address and CIDR mask, gateway is the next-hop router's IP Address and metric is the numerical weighting of the static route. Multiple static routes can be added by specifying `--add` with proper arguments multiple times. * `-r, --remove`: The static route(s) to remove, should be in the form of "ip/cidr gateway metric" Where ip/cidr is the destination IP address and CIDR mask, gateway is the next-hop router's IP Address and metric is the numerical weighting of the static route. Multiple static routes can be removed by specifying `--remove` with proper arguments multiple times. * `-s, --save-config`: If specified, the configuration will be saved. **NOTE** static routes specified in `--del` and `--add` must contain spaces and so must be quoted """ del_static_routes = [] if remove is None else remove add_static_routes = [] if add is None else add env = datapower.Environment(appliances, credentials, timeout=timeout) env.perform_action("ssh_connect", domain="default") atexit.register(env.perform_action, "ssh_disconnect") for appliance in env.appliances: issue_command = appliance.ssh_issue_command print appliance.hostname, "\n" print issue_command("config") print issue_command("interface {}".format(EthernetInterface)) print issue_command("show route") for static_route in del_static_routes: print issue_command("no ip route {}".format(static_route)) for static_route in add_static_routes: print issue_command("ip route {}".format(static_route)) print issue_command("exit") if save_config: print issue_command("write mem") print issue_command("y") print issue_command("show route") print issue_command("exit")