def download(): """ _function_: `mast.datapower.web.download()` This function is used to allow users to download ssh-transcripts. This function will accept a form POST with two fields: 1. `hostname`: The hostname of the appliance for which the transcript was generated. This is only used for creating the filename. 2. `content`: This is the content of the transcript and will be returned as a download for the user. Exposed at: `/download` via POST Parameters: This function accepts no arguments, but it does require a POST body to be present in the request. """ # This needs to be more universal. This is currently only useful for # ssh transcripts. This should allow any relevant file to be downloaded. if flask.request.method == "POST": t = Timestamp() hostname = flask.request.form.get("hostname") filename = '%s-%s-ssh-transcript.txt' % (t.timestamp, hostname) f = StringIO() f.write( unquote(flask.request.form.get("content")).replace( '\n', os.linesep).replace("+", " ")) f.seek(0) return flask.send_file(f, attachment_filename=filename, as_attachment=True)
def status(self): logger = make_logger("mast.datapower.status") t = Timestamp() check_hostname = "true" in flask.request.form.get( 'check_hostname').lower() appliances = flask.request.form.getlist('appliances[]') credentials = [ xordecode(_, key=xorencode( flask.request.cookies["9x4h/mmek/j.ahba.ckhafn"])) for _ in flask.request.form.getlist('credentials[]') ] if not appliances: return flask.abort(404) env = datapower.Environment(appliances, credentials, check_hostname=check_hostname) providers = flask.request.form.getlist("providers[]") resp = {"appliances": appliances, "time": t.short} for provider in providers: _provider = provider.split(".")[0] resp[provider] = [] for appliance in env.appliances: try: _status = appliance.get_status(_provider) except datapower.AuthenticationFailure: # This is to handle an intermittent authentication failure # sometimes issued by the DataPower. We will sleep 2 # seconds and try again sleep(2) try: return self.status() except: logger.exception( "An unhandled exception occurred during execution") raise except: logger.exception( "An unhandled exception occurred during execution") raise metric = _status.xml.find(PROVIDER_MAP[provider]).text resp[provider].append(metric) return flask.jsonify(resp)
def display_rows(header_row, rows, grep, provider): _format_string = "" for index, field in enumerate(header_row): _ = [field] + [str(x[index]) if len(x) >= index else " " for x in rows] length = len(max(_, key=len)) + 3 _format_string += "{%s: <%s}" % (index, length) clear() print Timestamp() print provider print "\n", _format_string.format(*header_row) print '-' * len(_format_string.format(*header_row)) for row in rows: for pattern in grep: if not pattern.search(_format_string.format(*row)): break else: print _format_string.format(*row)
def main(appliances=[], credentials=[], object_classes=[], domains=[], timeout=120, no_check_hostname=False, out_file="sample.xlsx", delim=os.linesep, by_appliance=False, no_prepend_timestamp=False, obfuscate_password=False): prepend_timestamp = not no_prepend_timestamp t = Timestamp() check_hostname = not no_check_hostname if by_appliance: logger.info("Generating workbooks by appliance") # we make the initial Environment to correctly handle credentials env = Environment(appliances, credentials, timeout, check_hostname=check_hostname) for appliance in env.appliances: logger.info("generating workbook for {}".format( appliance.hostname)) filename = os.path.split(out_file)[-1] filename = "{}-{}".format(appliance.hostname, filename) path = list(os.path.split(out_file)[:-1]) path.append(filename) _out_file = os.path.join(*path) logger.info("Workbook for {} will be stored in {}".format( appliance.hostname, _out_file)) _env = Environment([appliance.hostname], [appliance.credentials], timeout, check_hostname) create_workbook(_env, object_classes, domains, _out_file, delim, t, prepend_timestamp, obfuscate_password) else: logger.info("generating workbook") env = Environment(appliances, credentials, timeout, check_hostname=check_hostname) create_workbook(env, object_classes, domains, out_file, delim, t, prepend_timestamp, obfuscate_password)
def main(appliances=[], credentials=[], timeout=120, no_check_hostname=False, domains=["default"], providers=[], delay=0.5, out_file="./status.xlsx", by_appliance=False, no_prepend_timestamp=False): prepend_timestamp = not no_prepend_timestamp t = Timestamp() check_hostname = not no_check_hostname if by_appliance: logger.info("Generating workbooks by appliance") # we make the initial Environment to correctly handle credentials env = Environment(appliances, credentials, timeout, check_hostname=check_hostname) for appliance in env.appliances: logger.info("generating workbook for {}".format( appliance.hostname)) filename = os.path.split(out_file)[-1] filename = "{}-{}".format(appliance.hostname, filename) path = list(os.path.split(out_file)[:-1]) path.append(filename) _out_file = os.path.join(*path) logger.info("Workbook for {} will be stored in {}".format( appliance.hostname, _out_file)) _env = Environment([appliance.hostname], [appliance.credentials], timeout, check_hostname) create_workbook(_env, domains, providers, delay, _out_file, t, prepend_timestamp) else: logger.info("generating workbook") env = Environment(appliances, credentials, timeout, check_hostname=False) create_workbook(env, domains, providers, delay, out_file, t, prepend_timestamp)
def get_connections(env, clear_screen, class_filter, object_filter, domain_filter, state_filter, remote_ip_filter, remote_port_filter, local_ip_filter, local_port_filter): if clear_screen: clear() else: print "\n\n" for appliance in env.appliances: header = "LocalIP LocalPort RemoteIP RemotePort State Domain ServiceClass ServiceName".split( ) print "\n", appliance.hostname, "-", Timestamp(), "\n", "=" * 133 print "{0: <16} {1: <9} {2: <16} {3: <11} {4: <15} {5: <15} {6: <30} {7: <30}".format( *header) print "-" * 133 matching_values = [] tcp_table = appliance.get_status("TCPTable") for node in tcp_table.xml.findall(STATUS_XPATH): cls = node.find("serviceClass").text or "-" name = node.find("serviceName").text or "-" domain = node.find("serviceDomain").text or "-" state = node.find("state").text or "-" remote_ip = node.find("remoteIP").text or "-" remote_port = node.find("remotePort").text or "-" local_ip = node.find("localIP").text or "-" local_port = node.find("localPort").text or "-" values = [ local_ip, local_port, remote_ip, remote_port, state, domain, cls, name ] if class_filter.match(cls) and object_filter.match( name) and domain_filter.match( domain) and state_filter.match( state) and remote_ip_filter.match( remote_ip) and remote_port_filter.match( remote_port) and local_ip_filter.match( local_ip) and local_port_filter.match( local_port): matching_values.append(values) for value in matching_values: print "{0: <16} {1: <9} {2: <16} {3: <11} {4: <15} {5: <15} {6: <30} {7: <30}".format( *value)
def _call_method(func, kwargs): """Call func with kwargs if web is in kwargs, func should return a two-tupple containing (html, request_history). Here, we write the hsitory to a file and return the html for inclusion in the web GUI.""" import random random.seed() if "appliances" not in kwargs: pass elif not kwargs["appliances"][0]: # Kind of a hack to return the response we want in case no appliances # were checked in the gui def _func(*args, **kwargs): return ("Must select at least one appliance.", "Must select at least one appliance.") func = _func if "web" in kwargs: try: out, hist = func(**kwargs) except Exception, e: # The actions implemented should handle their own exceptions, # but if one makes it's way up here, we need to let the user know # part of that is suppressing the exception (because otherwise # we have no way of sending back the details) import traceback msg = "Sorry, an unhandled exception occurred while " msg += "performing action:\n\n\t {}".format(str(e)) out, hist = msg, traceback.format_exc() sys.stderr.write(traceback.format_exc()) t = Timestamp() # TODO: move this path to configuration filename = os.path.join("var", "www", "static", "tmp", "request_history", t.timestamp) if not os.path.exists(filename): os.makedirs(filename) rand = random.randint(10000, 99999) _id = "{}-{}.log".format(str(t.timestamp), str(rand)) filename = os.path.join(filename, _id) with open(filename, 'wb') as fout: fout.write(hist) return flask.Markup(out), _id
def upload(): """ _function_: `mast.datapower.web.upload()` This handles uploads of files, stores them in a temporary directory, and returns the path and filename in JSON format Exposed at: `/upload` via POST Parameters: This function does not accept any arguments, but does require a POST body to be present in the request. """ global upload_dir UPLOAD_DIRECTORY = upload_dir t = Timestamp() fin = flask.request.files["_file_in"] filename = os.path.join(UPLOAD_DIRECTORY, t.timestamp) os.makedirs(filename) filename = os.path.join(filename, fin.filename) fin.save(filename) return flask.jsonify({"filename": filename})
def restore_normal_backup(appliances=[], credentials=[], timeout=120, no_check_hostname=False, file_in=None, Domain="", source_type="ZIP", overwrite_files=True, overwrite_objects=True, rewrite_local_ip=True, deployment_policy=None, import_domain=True, reset_domain=True, dry_run=False, out_dir="tmp", web=False): """Restores a normal backup to the specified appliances and Domains. Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-f, --file-in`: The backup file which will be restored. This must be in the format specified in source_type * `-D, --Domain`: The domain to which to restore the backup * `-s, --source-type`: The type of backup, must be either "ZIP" or "XML" * `-N, --no-overwrite-files`: Whether to overwrite files when restoring the backup * `--no-overwrite-objects`: Whether to overwrite objects when restoring the backup * `--no-rewrite-local-ip`: Whether to rewrite the local IP Addresses * `-d, --deployment-policy`: The deployment policy to apply when restoring the backup * `--no-import-domain`: Whether we are importing a domain * `--no-reset-domain`: Whether to reset the domain * `--dry-run`: Whether this should be a dry-run * `-o, --out_dir`: (NOT NEEDED IN WEB GUI) The directory (local) where you would want all of the files generated by the restore to be placed * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" logger = make_logger("mast.backups") t = Timestamp() check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) logger.info( "Attempting to restore normal backup on {} in {} domain".format( str(env.appliances), Domain)) kwargs = { "file_in": file_in, "source_type": source_type, "domain": Domain, "overwrite_files": overwrite_files, "overwrite_objects": overwrite_objects, "rewrite_local_ip": rewrite_local_ip, "deployment_policy": deployment_policy, "import_domain": import_domain, "reset_domain": reset_domain, "dry_run": dry_run } resp = env.perform_action("restore_normal_backup", **kwargs) logger.debug("Responses received {}".format(str(resp))) out_dir = os.path.join(out_dir, "restore_normal_backup", t.timestamp) os.makedirs(out_dir) for host, r in resp.items(): filename = os.path.join( out_dir, "{}-{}-{}-results.xml".format(t.timestamp, host, Domain)) with open(filename, 'wb') as fout: fout.write(r.pretty) if web: return util.render_see_download_table(resp), util.render_history(env)
def get_secure_backup(appliances=[], credentials=[], timeout=1200, no_check_hostname=False, out_dir='tmp', CryptoCertificate="", destination='local:/raid0', include_iscsi=False, include_raid=False, remove=True, quiesce_before=True, unquiesce_after=True, quiesce_timeout=60, web=False): """Performs a secure backup of the specified domain. Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-o, --out-dir`: (NOT NEEDED IN WEB GUI) The directory (local) to store the backup * `-C, --CryptoCertificate`: The CryptoCertificate object to use to encrypt the backup * `-d, --destination`: The base location (on the appliance) to store the backup * `-i, --include-iscsi`: Whether to include the iscsi filesystem * `-I, --include-raid`: Whether to include the RAID filesystem * `-N, --no-remove`: If specified the backup will NOT be removed from the DataPower * `--no-quiesce-before`: If specified, the appliance will not be quiesced before performing the secure backup * `--no-unquiesce-after`: If specified, the appliance will not be unquiesced after performing the secure backup * `-q, --quiesce-timeout`: The timeout to wait before the appliance attempts to quiesce * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" logger = make_logger("mast.backups") check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) output = "" if quiesce_before: resp = {} for appliance in env.appliances: logger.info("Quiescing {} in preparation of Secure Backup".format( appliance.hostname)) resp[appliance.hostname] = appliance.QuiesceDP( timeout=quiesce_timeout) logger.debug("Response received {}".format( resp[appliance.hostname])) if web: output += util.render_boolean_results_table( resp, suffix="Quiesce_appliance") sleep(quiesce_timeout) t = Timestamp() if destination.endswith("/"): destination = destination.rstrip("/") destination = '%s/%s' % (destination, t.timestamp) kwargs = {'Dir': destination, 'domain': 'default'} logger.info("Creating directory {} on {} to store Secure Backup".format( destination, str(env.appliances))) resp = env.perform_async_action('CreateDir', **kwargs) logger.debug("Responses received {}".format(str(resp))) if web: output += util.render_boolean_results_table(resp, suffix="CreateDir") include_raid = 'on' if include_raid else 'off' include_iscsi = 'on' if include_iscsi else 'off' kwargs = { 'cert': CryptoCertificate, 'destination': destination, 'include_iscsi': include_iscsi, 'include_raid': include_raid } logger.info("Attempting to perform a Secure Backup on {}".format( str(env.appliances))) resp = env.perform_async_action('SecureBackup', **kwargs) logger.debug("Responses received: {}".format(str(resp))) if web: output += util.render_boolean_results_table(resp, suffix="SecureBackup") if web: results = {} remove_results = {} for appliance in env.appliances: directory = os.path.join(out_dir, appliance.hostname, "SecureBackup", t.timestamp) start = time() while not appliance.file_exists( '{}/backupmanifest.xml'.format(destination), 'default'): sleep(5) if time() - start > timeout: raise TimeoutError logger.info("Attempting to retrieve Secure Backup from {}".format( appliance.hostname)) appliance.copy_directory(destination, directory) _directory = os.path.join( directory, destination.replace(":", "").replace("///", "/")) try: logger.info("Attempting to verify Secure Backup for {}".format( appliance.hostname)) if appliance.verify_local_backup(_directory): logger.info("Secure Backup integrity verified for {}".format( appliance.hostname)) if web: results[appliance.hostname] = "Succeeded" else: print '\t', appliance.hostname, " - ", "Succeeded" if remove: logger.info( "Attempting to remove Secure Backup from appliance " "{}".format(appliance.hostname)) _resp = appliance.RemoveDir(Dir=destination, domain='default') logger.debug("Response received: {}".format(_resp)) if web: remove_results[appliance.hostname] = _resp else: logger.warn("Secure Backup for {} Corrupt!".format( appliance.hostname)) if web: results[appliance.hostname] = "Failed" else: print '\t', appliance.hostname, " - ", "Failed" appliance.log_error('Verification of backup in %s failed' % (_directory)) except: if web: results[appliance.hostname] = "Failed" logger.exception( "An unhandled exception occurred during execution.") if web: output += util.render_results_table(results, suffix="verify-SecureBackup") output += util.render_boolean_results_table(remove_results, suffix="RemoveDir") if unquiesce_after: resp = {} for appliance in env.appliances: logger.info("Attempting to unquiesce {}".format( str(appliance.hostname))) resp[appliance.hostname] = appliance.UnquiesceDP() logger.debug("Response received: {}".format( resp[appliance.hostname])) if web: output += util.render_boolean_results_table( resp, suffix="Unquiesce_appliance") if web: return output, util.render_history(env)
def get_normal_backup(appliances=[], credentials=[], timeout=120, no_check_hostname=False, Domain=[], comment="", out_dir='tmp', individual=False, web=False): """Performs a normal backup of the specified domain. Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-D, --Domain`: The domains to backup (all-domains will backup all domains) To spcify multiple domains, use multiple entries of the form `[-D domain1 [-D domain2...]]` * `-C, --comment`: The comment to add to the backup * `-o, --out-dir`: (NOT NEEDED IN WEB GUI) The directory (local) where you would like to store the backup * `-I, --individual`: If specified and all-domains is specified as --Domain then backup each domain individually instead of "all-domains" * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" logger = make_logger("mast.backups") t = Timestamp() check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) if not Domain: raise ValueError( "Must provide one or more domains including 'all-domains'") if isinstance(Domain, basestring): Domain = [Domain] # Fixes duplicate domains issue Domain = list(set(Domain)) results = {} if not individual: logger.info("Attempting to retrieve normal backup from " "{} in {} domain".format(str(env.appliances), Domain)) kwargs = {'domains': Domain, 'comment': comment} _results = env.perform_async_action('get_normal_backup', **kwargs) logger.debug("backups retrieved, check file for contents") for hostname, backup in _results.items(): directory = os.path.join(out_dir, hostname, "NormalBackup", t.timestamp) os.makedirs(directory) filename = os.path.join( directory, '%s-%s-%s.zip' % (t.timestamp, hostname, "_".join(Domain))) logger.debug("Writing backup for {} to {}".format( hostname, filename)) with open(filename, 'wb') as fout: fout.write(backup) if _verify_zip(filename): logger.info("backup for {} in {} domain verified".format( hostname, str(Domain))) results[hostname + "-" + "_".join(Domain) + "-normalBackup"] = "Verified" else: logger.info("backup for {} in {} domain corrupt".format( hostname, str(Domain))) results[hostname + "-" + "_".join(Domain) + "-normalBackup"] = "Corrupt" else: for domain in Domain: logger.info("Attempting to retrieve normal backup from " "{} in {} domain".format(str(env.appliances), domain)) kwargs = {'domains': domain, 'comment': comment} _results = env.perform_async_action('get_normal_backup', **kwargs) logger.debug("backups retrieved, check file for contents") for hostname, backup in _results.items(): directory = os.path.join(out_dir, hostname, "NormalBackup", t.timestamp) if not os.path.exists(directory): os.makedirs(directory) filename = os.path.join( directory, '%s-%s-%s.zip' % (t.timestamp, hostname, domain)) logger.debug("Writing backup for {} to {}".format( hostname, filename)) with open(filename, 'wb') as fout: fout.write(backup) if _verify_zip(filename): logger.info("backup for {} in {} domain verified".format( hostname, domain)) results[hostname + "-" + domain + "-normalBackup"] = "Verified" else: logger.info("backup for {} in {} domain corrupt".format( hostname, domain)) results[hostname + "-" + domain + "-normalBackup"] = "Corrupt" if web: return util.render_results_table(results), util.render_history(env) for k, v in results.items(): print print k print '=' * len(k) print v print
import os import openpyxl from time import sleep from mast.cli import Cli from mast.logging import make_logger from mast.datapower import datapower from mast.timestamp import Timestamp from mast.datapower.datapower import Environment t = Timestamp() logger = make_logger("mast.getstatus", propagate=False) def recurse_status(prefix, elem, row, header_row): for child in elem.findall(".//*"): logger.info("Found child {}".format(child.tag)) if child.tag not in header_row: logger.debug("Appending {} to header row".format(child.tag)) header_row.append(child.tag) if len(child): _prefix = "{}/{}".format(prefix, child.tag) logger.info("child {} has children, recursing...".format(_prefix)) recurse_status(_prefix, child, row, header_row) else: logger.info("Adding {} -> {}".format(child.tag, child.text)) row.insert(header_row.index(child.tag), child.text) def create_workbook(env, domains, providers, delay, out_file, timestamp, prepend_timestamp):
def make_logger( name, level=level, fmt=_format, max_bytes=max_bytes, backup_count=backup_count, delay=delay, propagate=propagate, ): """ _function_: `mast.logging.make_logger(name, level=level, fmt=_format, filename=None, when=unit, interval=interval, propagate=propagate, backup_count=backup_count)` Returns an instance of logging.Logger configured with a [logging.handlers.TimedRotatingFileHandler](https://docs.python.org/2/library/logging.handlers.html#timedrotatingfilehandler) handler. Arguments passed to this function determine the format, level, filename, time unit, interval, backup count and whether to propagate messages to parent loggers (defined by dot seperated heirarchy ie in `mast.datapower`, `datapower` is a logger with a parent logger of mast). Parameters: * `name`: Required. the name of the logger instance. This follows conventions mentioned [here](https://docs.python.org/2/library/logging.html#logger-objects) * `level`: The logging level to listen for. Accepts an `int` or one of the logging modules convenience constants defined [here](https://docs.python.org/2/library/logging.html#logging-levels) * `fmt`: The format of the log message, see [here](https://docs.python.org/2/library/logging.html#formatter-objects) and [here](https://docs.python.org/2/library/logging.html#logrecord-attributes) for more details * `filename`: The filename to log to. Defaults to the name of the logger appended with `.log` in the `$MAST_HOME/var/log` directory or `$MAST_HOME/var/log/mastd` directory if running as `mastd` * `when`: The time unit to use for rolling over the log file as detailed [here](https://docs.python.org/2/library/logging.handlers.html#timedrotatingfilehandler) * `interval`: The number of time units to wait before rolling the log files as detailed [here](https://docs.python.org/2/library/logging.handlers.html#timedrotatingfilehandler) * `propagate`: Whether to propagate log messages up the ancestry chain (ie. if you have a logger `mast.datapower`, and propagate is set to `False` messages sent to the this logger will not be propagated to the `mast` logger). See [here](https://docs.python.org/2/library/logging.html#logging.Logger.propagate) for more details. * `backup_count`: The number of "rolled" log files to keep, see [here](https://docs.python.org/2/library/logging.handlers.html#timedrotatingfilehandler) for more details. Usage: :::python from mast.logging import make_logger logger = make_logger("my_module") logger.info("informational message") logger.debug("debug message") """ t = Timestamp() _logger = logging.getLogger(name) if _logger.handlers: if len(_logger.handlers) >= 1: return _logger _logger.setLevel(level) _formatter = logging.Formatter(fmt) pid = os.getpid() directory = os.path.join( mast_home, "var", "log", getpass.getuser(), "{}-{}".format(t.timestamp, str(pid)), ) filename = os.path.join( directory, "{}-{}.log".format(name, t.timestamp), ) if not os.path.exists(directory): os.makedirs(directory) _handler = RotatingFileHandler( filename, maxBytes=max_bytes, backupCount=backup_count, delay=delay, ) _handler.setFormatter(_formatter) _handler.setLevel(level) _logger.addHandler(_handler) _logger.propagate = propagate return _logger
def clean_up(appliances=[], credentials=[], timeout=120, no_check_hostname=False, Domain='default', checkpoints=False, export=False, error_reports=False, recursive=False, logtemp=False, logstore=False, backup_files=True, out_dir='tmp', web=False): """This will clean up the specified appliances filesystem optionally (defaults to True) taking copies of the files as backups. Parameters: * `-a, --appliances` - The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-D, --Domain`: The domain who's filesystem you would like to clean up * `-C, --checkpoints`: If specified, all checkpoints will be removed from the domain * `-e, --export`: If specified all exports will be removed from the domain * `-l, --logtemp`: If specified, all files in `logtemp:` will be removed from the domain * `-L, --logstore`: If specified, all files in `logstore:` will be removed * `-E, --error-reports`: If specified, all error reports will be removed from the appliance(s) * `-r, --recursive`: If specified, directories will be cleaned recursively * `--no-backup-files`: If specified, files will not be backed up before deleting * `-o, --out-dir`: The directory to save backed up files * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) t = Timestamp() dirs = [] if checkpoints: dirs.append('chkpoints:/') if export: dirs.append('export:/') if logtemp: dirs.append('logtemp:/') if logstore: dirs.append('logstore:/') if web: rows = [] for appliance in env.appliances: if web: rows.append((appliance.hostname, )) for _dir in dirs: _clean_dir(appliance, _dir, Domain, recursive, backup_files, t.timestamp, out_dir) if web: rows.append(("", _dir, "Cleaned")) if error_reports: _clean_error_reports(appliance, Domain, backup_files, t.timestamp, out_dir) rows.append(("", "ErrorReports", "Cleaned")) return flask.render_template( "results_table.html", header_row=["Appliance", "Location", "Action"], rows=rows), util.render_history(env)
def call_method(plugin, form): """Gather the arguments and function name from form then invoke _call_method. Wrap the results in html and return them.""" t = Timestamp() name = form.get("callable") arguments, func = _get_arguments(plugin, name) kwargs = {} for arg, default in arguments: if isinstance(default, bool): if arg == "web": kwargs[arg] = True continue value = form.get(arg) if value == 'true': kwargs[arg] = True else: kwargs[arg] = False elif isinstance(default, list): # TODO: This needs to implement a selection feature if arg == 'appliances': kwargs[arg] = form.getlist(arg + '[]') elif arg == 'credentials': kwargs[arg] = [ xordecode( _, key=xorencode( flask.request.cookies["9x4h/mmek/j.ahba.ckhafn"])) for _ in form.getlist(arg + '[]') ] else: kwargs[arg] = form.getlist(arg + '[]') elif isinstance(default, basestring): if arg == 'out_dir': kwargs[arg] = os.path.join('tmp', 'web', name, t.timestamp) elif arg == 'out_file' and default is not None: kwargs[arg] = os.path.join( "tmp", "web", name, "{}-{}{}".format(t.timestamp, name, os.path.splitext(default)[1])).replace( os.path.sep, "/") else: kwargs[arg] = form.get(arg) or default elif isinstance(default, int): kwargs[arg] = int(form.get(arg)) or default elif default is None: kwargs[arg] = form.get(arg) or default out, history_id = _call_method(func, kwargs) link = "" if 'out_dir' in kwargs: config = get_config("server.conf") static_dir = config.get('dirs', 'static') fname = "" for appliance in kwargs['appliances']: fname = "{}-{}".format(fname, appliance) fname = "{}-{}{}.zip".format(t.timestamp, name, fname) zip_filename = os.path.join(static_dir, 'tmp', fname) zip_file = zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) _zipdir(kwargs['out_dir'], zip_file) zip_file.close() #filename = '%s-%s.zip' % (t.timestamp, name) link = flask.Markup(flask.render_template('link.html', filename=fname)) if 'out_file' in kwargs and kwargs["out_file"] is not None: import shutil config = get_config("server.conf") static_dir = config.get('dirs', 'static') dst = os.path.join(static_dir, "tmp", os.path.basename(kwargs["out_file"])) shutil.copyfile(kwargs["out_file"], dst) link = flask.Markup( flask.render_template('link.html', filename=os.path.basename( kwargs["out_file"]))) out = flask.render_template('output.html', output=out, callable=name, timestamp=str(t), history_id=history_id, link=link) if 'out_dir' in kwargs: out = out return out
dst_dir = os.path.join(dst, directory) if not os.path.exists(dst_dir): os.mkdir(dst_dir) for f in files: print os.path.join(root, f), "->", dst shutil.copy(os.path.join(root, f), dst) print "\n\nhotfix installed" print "\nTo re-generate the latest documentation, please run" print "\t{}mast contrib/gendocs.py".format(mast_home + os.path.sep) if __name__ == "__main__": main() ''' now = Timestamp().timestamp mast_home = os.environ["MAST_HOME"] default_out_file = os.path.join( mast_home, "tmp", "hotfix.{}.zip".format(now)) default_build_dir = os.path.join( mast_home, "tmp", "{}-hotfix-build".format(now)) repos = [ "https://github.com/McIndi/mast.cli/archive/master.zip", "https://github.com/McIndi/mast.config/archive/master.zip",
def main(appliances=[], credentials=[], timeout=120, no_check_hostname=False, domains=["default"], providers=[], object_classes=[], delim=os.linesep, delay=0.5, out_file="./status.xlsx", by_appliance=False, no_prepend_timestamp=False, obfuscate_password=False, persisted=False): r""" _Script_: `contrib/dp-query.py` DESCRIPTION: This script will create an excel workbook generated from the output from the specified appliances for arbitrary status providers and/or object configuration. Each status provider and object class configuration will be contained in a seperate worksheet RETURNS: An Excel workbook USAGE: ```bash $ mast contrib/dp-query --help $ mast contrib/dp-query --appliances APPLIANCE --credentials CREDS --domains all-domains --providers ServicesStatusPlus --object-classes MultiProtocolGateway HTTPFrontSideHandler ``` PARAMETERS: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-d, --domains`: The domains to query. To pass multiple arguments to this parameter, use multiple entries of the form `[-d domain1 [-d domain2...]]`. If `all-domains` is passed to this parameter, all domains will be queried. * `-p, --providers`: The status providers to query. To pass multiple arguments to this parameter, use multiple entries of the form `[-p provider1 [-p provider2...]]`. * `-o, --object-classes`: The object classes to pull the configuration of. To pass multiple arguments to this parameter, use multiple entries of the form `[-o objectclass1 [-o objectclass2...]]`. * `-D, --delim`: The delimeter to use when multiple values will end up in the same cell. Defaults to your system's default line termination character (`\r\n` on windows or `\n` on Linux) * `--delay`: The amount of time (in seconds) to wait between requests to the appliances. This can be extremely useful if you need to avoid over-working the appliance for example if you need to gather information during peak periods you would probably want to set this to one or more seconds. Defaults to `0.5`. * `-O, --out-file`: The path and filename of the output file. This will be an excel workbook, so it should have the extention `.xlsx` * `-b, --by-appliance`: If specified, a seperate excel workbook will be generated for each appliance. * `-N, --no-prepend-timestamp`: If specified, this script will not prepend a timestamp to the output filename * `--obfuscate-password`: If specified, any object configurations which contain a password will have its value replaced with "********", that's eight asterics. * `-P, --persisted`: If specified, the persisted configuration of any specified object classes will be retrieved instead of the running configuration """ prepend_timestamp = not no_prepend_timestamp t = Timestamp() check_hostname = not no_check_hostname if by_appliance: logger.info("Generating workbooks by appliance") # we make the initial Environment to correctly handle credentials env = Environment(appliances, credentials, timeout, check_hostname=check_hostname) for appliance in env.appliances: logger.info("generating workbook for {}".format( appliance.hostname)) filename = os.path.split(out_file)[-1] filename = "{}-{}".format(appliance.hostname, filename) path = list(os.path.split(out_file)[:-1]) path.append(filename) _out_file = os.path.join(*path) logger.info("Workbook for {} will be stored in {}".format( appliance.hostname, _out_file)) _env = Environment([appliance.hostname], [appliance.credentials], timeout, check_hostname) create_workbook(_env, domains, providers, object_classes, delay, delim, _out_file, t, prepend_timestamp, obfuscate_password, persisted) else: logger.info("generating workbook") env = Environment(appliances, credentials, timeout, check_hostname=False) create_workbook(env, domains, providers, object_classes, delay, delim, out_file, t, prepend_timestamp, obfuscate_password, persisted)
def postdeploy(appliances=[], credentials=[], timeout=120, no_check_hostname=False, Domain="", unquiesce_domain=True, unquiesce_appliance=False, postdeploy_command=None, save_config=True, web=False): """This is a simple script which will allow you to unquiesce your domain or appliances after you quiesce them for a deployment. Also this will allow you to save the config. Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-D, --Domain`: The domain which will be unquiesced and persisted. * `-N, --no-unquiesce-domain`: If specified, this script will not attempt to unquiesce the domain * `-u, --unquiesce-appliance`: If specified, this script will attempt to unquiesce the appliance * `-p, --postdeploy-command`: This command will be "shelled out" to the machine running MAST after unquiescing and saving the configuration, use this parameter to clean up VCS artifacts and/or perform check-outs of your deployed services or similar operations * `--no-save-config`: If specified, the configuration will not be saved in the application domain * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" import mast.datapower.system as system check_hostname = not no_check_hostname logger = make_logger('mast.datapower.deployment') env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) if web: output = "" history = "" for appliance in env.appliances: if unquiesce_appliance: appliance.log_info("Attempting to unquiesce appliance") _out = system.unquiesce_appliance( appliances=[appliance.hostname], credentials=[appliance.credentials], timeout=timeout, no_check_hostname=no_check_hostname, web=web) appliance.log_info("Finished Unquiescing appliance") if web: output += _out[0] history += _out[1] else: print "Finished unquiescing appliance" if unquiesce_domain: appliance.log_info("Attempting to unquiesce domain") _out = system.unquiesce_domain(appliances=[appliance.hostname], credentials=[appliance.credentials], timeout=timeout, no_check_hostname=no_check_hostname, Domain=Domain, web=web) appliance.log_info("Finished Unquiescing domain") if web: output += _out[0] history += _out[1] else: print "Finished unquiescing domain" if save_config: appliance.log_info( "Attempting to save configuration after deployment") _out = system.save_config(appliances=[appliance.hostname], credentials=[appliance.credentials], timeout=timeout, no_check_hostname=no_check_hostname, Domain=Domain, web=web) appliance.log_info( "Finished saving configuration after deployment") if web: output += _out[0] history += _out[1] else: print "Finished saving the configuration" if postdeploy_command: logger.info( "Post-Deployment command '{}' found. Executing at {}".format( postdeploy_command, str(Timestamp()))) out, err = system_call(command=postdeploy_command) out = str(out) err = str(err) logger.info( "finished executing Post-Deployment command '{}' at {}., output: {}" .format(postdeploy_command, str(Timestamp()), ";".join([out, err]))) if web: from mast.plugin_utils.plugin_utils import render_results_table results = { "postdeploy command": "{}\n\nout: {}\n\nerr: {}".format(postdeploy_command, out, err) } output += render_results_table(results) else: print "Finished running post-deploy command. output: {}".format( ";".join([out, err])) if web: return output, history
def deploy(appliances=[], credentials=[], timeout=180, no_check_hostname=False, Domain=[], file_in=None, deployment_policy="", dry_run=False, overwrite_files=True, overwrite_objects=True, rewrite_local_ip=True, object_audit=True, out_dir='tmp', format='ZIP', quiesce_domain=True, quiesce_appliance=False, quiesce_timeout=120, web=False): """Perform a deployment/migration of a service/object to an IBM DataPower appliance. This script will try to perform the deployment/migration in a manner consistent with best practices. __WARNING__: There is an inherent security risk involved in this script, in order to allow the most flexible integration with various Version Control Systems possible, we allow a pre-deployment hook and a post-deployment hook which will be "shelled out" to your operating system. For this reason PLEASE be sure to run this script (and the MAST Web GUI server) as a user with appropriate permissions. DO NOT RUN AS ROOT!!! Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-D, --Domain`: The domain to import the configuration into, also the domain to quiesce if `--no-quiesce-domain` is not specified. * `-f, --file-in`: This is the configuration file that you are importing. It must be in the format specified by the format parameter. * `-d, --deployment-policy`: The deployment policy to apply to the import, must already exist on the appliance * `--dry-run`: If specified, a dry-run will be performed instead of an actual import * `-N, --no-overwrite-files`: If specified, no files will be over- written during the import * `--no-overwrite-objects`: If specified, no objects will be over- written during import * `--no-rewrite-local-ip`: If specified local ip addresses will not be rewritten on import * `--no-object-audit`: If specified, an object audit will not be performed. An object audit is a diff between the running and persisted configuration * `-o, --out-dir`: This is where to place the artifacts generated by this script * `-F, --format`: The format of the configuration file, must be either "ZIP" or "XML". * `--no-quiesce-domain`: If specified, the domain will not be quiesced prior to the deployment. * `-q, --quiesce-appliance`: If specified, the appliance will be quiesced prior to the deployment * `-Q, --quiesce-timeout`: This is the amount of time for the appliance to wait before beginning the quiescence procedure. * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" import mast.datapower.system as system from mast.datapower.developer import _import logger = make_logger('mast.datapower.deployment') if web: output = "" history = "" check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) for appliance in env.appliances: appliance.log_info("Deployment started on {}".format( appliance.hostname)) # Quiesce Domain if quiesce_domain: logger.info("Quiescing domain {} before deployment at {}".format( Domain, str(Timestamp()))) _out = system.quiesce_domain(appliances=[appliance.hostname], credentials=credentials, timeout=timeout, no_check_hostname=no_check_hostname, Domain=Domain, quiesce_timeout=quiesce_timeout, web=web) logger.info( "Finished quiescing domain {} before deployment at {}".format( Domain, str(Timestamp()))) sleep(quiesce_timeout) if web: output += _out[0] history += _out[1] # Quiesce Appliance if quiesce_appliance: logger.info("Quiescing appliances before deployment at {}".format( str(Timestamp()))) _out = system.quiesce_appliance( appliances=appliance.hostname, credentials=credentials, timeout=timeout, no_check_hostname=no_check_hostname, quiesce_timeout=quiesce_timeout, web=web) logger.info( "Finished quiescing appliances before deployment at {}".format( str(Timestamp()))) sleep(quiesce_timeout) if web: output += _out[0] history += _out[1] appliance.log_info("Attempting to import configuration at '{}'".format( str(Timestamp()))) file_out = os.path.join( out_dir, '{}-deployment_results.txt'.format(appliance.hostname)) # import configuration _out = _import(appliances=[appliance.hostname], credentials=credentials, timeout=timeout, no_check_hostname=no_check_hostname, Domain=Domain, file_in=file_in, deployment_policy=deployment_policy, dry_run=dry_run, overwrite_files=overwrite_files, overwrite_objects=overwrite_objects, rewrite_local_ip=rewrite_local_ip, source_type=format, out_dir=out_dir, web=web) if web: output += _out[0] history += _out[1] appliance.log_info("Finished importing configuration at {}".format( str(Timestamp()))) # unquiesce domain if quiesce_domain: appliance.log_info("Attempting to unquiesce domain") _out = system.unquiesce_domain(appliances=[appliance.hostname], credentials=credentials, timeout=timeout, no_check_hostname=no_check_hostname, Domain=Domain, web=web) appliance.log_info("Finished unquiescing domain") if web: output += _out[0] history += _out[1] # unquiesce appliance if quiesce_appliance: logger.info("Quiescing appliances before deployment at {}".format( str(Timestamp()))) _out = system.unquiesce_appliance( appliances=[appliance.hostname], credentials=credentials, timeout=timeout, no_check_hostname=no_check_hostname, web=web) logger.info( "Finished quiescing appliances before deployment at {}".format( str(Timestamp()))) if web: output += _out[0] history += _out[1] if object_audit: appliance.log_info( "Post-Deployment Object audit started at {}".format( str(Timestamp()))) _out = system.objects_audit(appliances=[appliance.hostname], credentials=credentials, timeout=timeout, no_check_hostname=no_check_hostname, out_dir=out_dir, web=web) appliance.log_info( "Post-Deployment Object audit finished at {}".format( str(Timestamp()))) if web: output += _out[0] history += _out[1] if web: return output, history
def predeploy(appliances=[], credentials=[], timeout=120, no_check_hostname=False, out_dir="tmp", Domain="", comment="", predeploy_command=None, CryptoCertificate="", secure_backup_destination="local:/raid0", backup_default=True, backup_all=True, do_secure_backup=False, do_normal_backup=True, set_checkpoints=True, include_iscsi=False, include_raid=False, remove_secure_backup=True, default_checkpoint=True, remove_oldest_checkpoint=True, allow_shell=False, web=False): """Perform routine pre-deployment actions. Everything is optional, but if you wish to perform an action, you must provide the necessary arguments. Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-o, --out-dir`: (Not needed in web GUI) The directory (local) where you would like to store the artifacts generated by this script * `-D, --Domain`: This is the app domain to backup and the app domain in which to set a checkpoint * `-C, --comment`: This is shared among other actions. The comment is used to build the name of the checkpoint. * `-p, --predeploy-command`: This command will be "shelled out" to the machine running MAST after performing the backups and checkpoints. Use this parameter to pull from version control or similar operations. * `--CryptoCertificate`: The CryptoCertificate with which to encrypt the secure backup * `-s, --secure-backup-destination`: The destination (on the DataPower) where the secure backup will be stored * `-N, --no-backup-default`: Whether to also backup the default domain * `--no-backup-all`: Whether to also backup all-domains * `-d, --do-secure-backup`: Whether to retrieve a secure backup * `--no-do-normal-backup`: Whether to retrieve normal backups * `--no-set-checkpoints`: Whether to set checkpoints * `-i, --include-iscsi`: Whether to include the iscsi volume in the secure backup * `-I, --include-raid`: Whether to include the raid volume in the secure backup * `--no-remove-secure-backup`: Whether to remove the secure backup from the appliance after verifying your local copy. * `--no-default-checkpoint`: Whether to create a checkpoint in the default domain * `--no-remove-oldest-checkpoint`: Whether to remove the oldest checkpoint from the domain IF AND ONLY IF the maximum number of checkpoints has been reached. * `-A, --allow-shell`: Whether to allocate a shell for the execution of the predeploy-command * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__ Here is what is possible (will be done in this order): 1. Secure Backup * The following parameters apply: * `-d, --do-secure-backup` * `-o, --out-dir` * `--CryptoCertificate` * `-s, --secure-backup-destination` * `-i, --include-iscsi` * `-I, --include-raid` * `--no-remove-secure-backup` 2. Normal Backups * The following parameters apply: * `--no-do-normal-backup` * `-o, --out-dir` * `-D, --Domain` * `-N, --no-backup-default` * `--no-backup-all` * `-C, --comment` 3. Checkpoints * The following parameters apply: * `--no-set-checkpoints` * `-D, --Domain` * `-C, --comment` * `--no-default-checkpoint` * `--no-remove-oldest-checkpoint`""" from mast.datapower.backups import set_checkpoint from mast.datapower.backups import get_normal_backup, get_secure_backup logger = make_logger('mast.datapower.deployment') check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) if web: output = "" history = "" # Loop through appliances so we will only affect one appliance at a time for appliance in env.appliances: # Secure Backup if do_secure_backup: if not CryptoCertificate: # Fail if CryptoCertificate is not given logger.error("Cert must be specified in order " "to perform a secure backup!") sys.exit(-1) logger.info("Starting Secure Backup for {}".format( appliance.hostname)) _out = get_secure_backup(appliances=appliance.hostname, credentials=appliance.credentials, timeout=timeout, out_dir=out_dir, CryptoCertificate=CryptoCertificate, destination=secure_backup_destination, include_iscsi=include_iscsi, include_raid=include_raid, remove=remove_secure_backup, quiesce_before=False, unquiesce_after=False, no_check_hostname=no_check_hostname, web=web) if web: output += _out[0] history += _out[1] logger.info("Finished Secure Backup for {}".format( appliance.hostname)) # Normal backups if do_normal_backup: logger.info("Pre-Deployment backups started at {}".format( str(Timestamp()))) domains = [Domain] if backup_default: domains.append("default") if backup_all: domains.append("all-domains") _out = get_normal_backup(appliances=[appliance.hostname], credentials=credentials, timeout=timeout, no_check_hostname=no_check_hostname, Domain=domains, comment=comment, out_dir=out_dir, web=web) logger.info("Pre-Deployment backups finished at {}".format( str(Timestamp()))) if web: output += _out[0] history += _out[1] # Checkpoints if set_checkpoints: logger.info("Pre-Deployment checkpoints started at {}".format( str(Timestamp()))) domains = [Domain] if default_checkpoint: domains.append("default") _out = set_checkpoint(appliances=[appliance.hostname], credentials=credentials, timeout=timeout, no_check_hostname=no_check_hostname, Domain=domains, comment=comment, remove_oldest=remove_oldest_checkpoint, web=web) logger.info("Pre-Deployment checkpoints finished at {}".format( str(Timestamp()))) if web: output += _out[0] history += _out[1] if predeploy_command: logger.info( "Pre-Deployment command '{}' found. Executing at {}".format( predeploy_command, str(Timestamp()))) out, err = system_call(command=predeploy_command, shell=allow_shell) out = str(out) err = str(err) logger.info( "finished executing Pre-Deployment command '{}' at {}., output: {}" .format(predeploy_command, str(Timestamp()), ";".join([out, err]))) if web: from mast.plugin_utils.plugin_utils import render_results_table results = { "predeploy command": "{}\n\nout: {}\n\nerr: {}".format(predeploy_command, out, err) } output += render_results_table(results) else: print "Finished running pre-deploy command. output: {}".format( ";".join([out, err])) if web: return output, history
def set_checkpoint(appliances=[], credentials=[], timeout=120, no_check_hostname=False, Domain=['default'], comment='', remove_oldest=True, web=False): """Sets a checkpoint in the given domains on the specified appliances Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-D, --Domain`: Domains to set checkpoints in. To spcify multiple domains, use multiple entries of the form `[-D domain1 [-D domain2...]]` * `-C, --comment`: The comment to use for the checkpoint (will also be used to build the checkpoint name) * `-N, --no-remove-oldest`: If specified this script will attempt to remove the oldest checkpoint __IF__ the maximum number of checkpoints exist * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" logger = make_logger("mast.backups") check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) logger.info("Attempting to set checkpoint on {} in {} domain(s)".format( str(env.appliances), str(Domain))) t = Timestamp() if web: header_row = ("Appliance", "Result") rows = [] for appliance in env.appliances: if not web: print appliance.hostname _domains = Domain print Domain print _domains if "all-domains" in _domains: _domains = appliance.domains print _domains for domain in _domains: print domain if not web: print "\t", domain name = '{0}-{1}-{2}'.format(comment, domain, t.timestamp) logger.debug( "Attempting to set checkpoint {} on {} in {} domain".format( name, appliance, domain)) if remove_oldest: _max = appliance.max_checkpoints(domain) if len(appliance.get_existing_checkpoints(domain)) >= _max: logger.info( "Maximum number of checkpoints for domain " "{} on {} reached. Removing oldest checkpoint.".format( domain, appliance.hostname)) _resp = appliance.remove_oldest_checkpoint(domain) logger.debug("Response received: {}".format(_resp)) kwargs = {'domain': domain, 'ChkName': name} resp = appliance.SaveCheckpoint(**kwargs) logger.debug("Response received: {}".format(resp)) if not web: if resp: print "\t\tSuccessful" else: print "\t\tFailed" if web: if resp: rows.append( ("{}-{}-set_checkpoint".format(appliance.hostname, domain), "Succeeded")) else: rows.append( ("{}-{}-set_checkpoint".format(appliance.hostname, domain), "Failed")) if web: return flask.render_template("results_table.html", header_row=header_row, rows=rows), util.render_history(env)
def _import(appliances=[], credentials=[], timeout=120, no_check_hostname=False, Domain=[], file_in=None, deployment_policy=None, deployment_policy_variables=None, dry_run=False, overwrite_files=True, overwrite_objects=True, rewrite_local_ip=True, source_type='ZIP', out_dir="tmp/", web=False): """Import a service/object into the specified appliances Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-D. --Domain`: The domain into which the configuration will be imported * `-f, --file-in`: The file to import into the specified domain. This __MUST__ match the format specified in source_type * `-d, --deployment-policy`: The deployment policy to use for the import (must already exist on the appliances) * `--dry-run`: Whether to do a dry-run (nothing will be imported) * `-N, --no-overwrite-files`: If specified, no files will be overwritten as part of the import * `--no-overwrite-objects`: If specified, no objects will be overwritten as part of the import * `--no-rewrite-local-ip`: If specified, no local ip addresses will be rewritten as part of the import * `-s, --source-type`: The type of file to import. Can be "XML" or "ZIP" * `-o, --out-dir`: The directory to output artifacts generated by this script * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" logger = make_logger("mast.developer") t = Timestamp() check_hostname = not no_check_hostname env = datapower.Environment( appliances, credentials, timeout, check_hostname=check_hostname) msg = "Attempting to import {} to {}".format(file_in, str(env.appliances)) logger.info(msg) if not web: print msg results = {} out_dir = os.path.join(out_dir, "import_results", t.timestamp) os.makedirs(out_dir) for appliance in env.appliances: if not web: print appliance.hostname results[appliance.hostname] = {} domains = Domain if "all-domains" in domains: domains = appliance.domains for domain in domains: if not web: print "\t", domain kwargs = { 'domain': domain, 'zip_file': file_in, 'deployment_policy': deployment_policy, 'deployment_policy_variables': deployment_policy_variables, 'dry_run': dry_run, 'overwrite_files': overwrite_files, 'overwrite_objects': overwrite_objects, 'rewrite_local_ip': rewrite_local_ip, 'source_type': source_type} resp = appliance.do_import(**kwargs) results[appliance.hostname][domain] = resp if not web: pprint_xml(resp.xml) logger.debug("Response received: {}".format(str(resp))) filename = os.path.join( out_dir, "{}-{}-import_results.xml".format( appliance.hostname, domain ) ) with open(filename, 'wb') as fout: fout.write(resp.pretty) if web: return util.render_see_download_table( results, suffix="import"), util.render_history(env)
def export(appliances=[], credentials=[], timeout=120, no_check_hostname=False, Domain="", object_name=None, object_class=None, comment='', format='ZIP', persisted=True, all_files=True, referenced_files=True, referenced_objects=True, out_dir='tmp', web=False): """Exports a service or object to be used to import into another domain or appliance Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-D, --Domain`: The domain from which to export service/object * `-o, --object-name`: The name of the object to export * `-O, --object-class`: The class of the object to export * `-C, --comment`: The comment to embed into the export * `-f, --format`: the format in which to export the configuration. This can be either "XML" or "ZIP" * `-N, --no-persisted`: If specified, the running configuration will be exported as opposed to the persisted configuration * `--no-all-files`: If specified, the export will not include all files * `--no-referenced-files`: If specified, the referenced files will not be included in the export * `--no-referenced-objects`: If specified, referenced objects will not be included in the export. * `--out-dir`: (**NOT NEEDED IN THE WEB GUI**)The directory (local) in which to save the export * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" logger = make_logger("mast.developer") t = Timestamp() if object_name is None or object_class is None: try: raise TypeError("Must Provide both object name and object class") except: logger.exception("Must Provide both object name and object class") raise check_hostname = not no_check_hostname env = datapower.Environment( appliances, credentials, timeout, check_hostname=check_hostname) msg = "Attempting to export {} from {}".format(object_name, str(env.appliances)) logger.info(msg) kwargs = { 'domain': Domain, 'obj': object_name, 'object_class': object_class, 'comment': comment, 'format': format, 'persisted': persisted, 'all_files': all_files, 'referenced_objects': referenced_objects, 'referenced_files': referenced_files} results = env.perform_action( 'export', **kwargs) for hostname, _export in results.items(): d = os.path.join(out_dir, hostname, t.timestamp) os.makedirs(d) extention = format.lower() filename = os.path.join(d, '%s-%s-%s.%s' % ( t.timestamp, hostname, object_name, extention)) msg = "Writing export of {} from {} to {}".format(object_name, hostname, filename) logger.debug(msg) if not web: print msg with open(filename, 'wb') as fout: fout.write(_export) if web: return util.render_see_download_table( results, suffix="export"), util.render_history(env)
from mast.timestamp import Timestamp from mast.datapower import datapower from mast.logging import make_logger from cStringIO import StringIO from pygments import highlight import mast.pprint as pprint from mast.cli import Cli import colorama import urllib2 import os import re colorama.init() default_base_dir = os.path.join(os.environ["MAST_HOME"], "tmp", "config") default_comment = "Update {}".format(Timestamp().friendly) def main(appliances=[], credentials=[], timeout=120, no_check_hostname=False, base_dir=default_base_dir, comment=default_comment, persisted=False, recursive=False, no_strip_timestamp=False, page=False): """ track_getconfig.py
def get_file(appliances=[], credentials=[], timeout=120, no_check_hostname=False, location=None, Domain='default', out_dir='tmp', web=False): """Retrieves a file from the specified appliances Parameters: * `-a, --appliances`: The hostname(s), ip address(es), environment name(s) or alias(es) of the appliances you would like to affect. For details on configuring environments please see the comments in `environments.conf` located in `$MAST_HOME/etc/default`. For details on configuring aliases, please see the comments in `hosts.conf` located in `$MAST_HOME/etc/default`. To pass multiple arguments to this parameter, use multiple entries of the form `[-a appliance1 [-a appliance2...]]` * `-c, --credentials`: The credentials to use for authenticating to the appliances. Should be either one set to use for all appliances or one set for each appliance. Credentials should be in the form `username:password`. To pass multiple credentials to this parameter, use multiple entries of the form `[-c credential1 [-c credential2...]]`. When referencing multiple appliances with multiple credentials, there must be a one-to-one correspondence of credentials to appliances: `[-a appliance1 [-a appliance2...]] [-c credential1 [-c credential2...]]` If you would prefer to not use plain-text passwords, you can use the output of `$ mast-system xor <username:password>`. * `-t, --timeout`: The timeout in seconds to wait for a response from an appliance for any single request. __NOTE__ Program execution may halt if a timeout is reached. * `-n, --no-check-hostname`: If specified SSL verification will be turned off when sending commands to the appliances. * `-l, --location`: The location of the file (on DataPower) you would like to get * `-D, --Domain`: The domain from which to get the file * `-o, --out-dir`: (NOT NEEDED IN THE WEB GUI)The directory you would like to save the file to * `-w, --web`: __For Internel Use Only, will be removed in future versions. DO NOT USE.__""" t = Timestamp() check_hostname = not no_check_hostname env = datapower.Environment(appliances, credentials, timeout, check_hostname=check_hostname) kwargs = {'domain': Domain, 'filename': location} responses = env.perform_async_action('getfile', **kwargs) if not os.path.exists(out_dir) or not os.path.isdir(out_dir): os.makedirs(out_dir) for hostname, fin in list(responses.items()): filename = location.split('/')[-1] filename = os.path.join(out_dir, '%s-%s-%s' % (hostname, t.timestamp, filename)) with open(filename, 'wb') as fout: fout.write(fin) if web: return util.render_see_download_table( responses, suffix="get_file"), util.render_history(env)