def _http_request(url, request_timeout=None):
"""
PRIVATE METHOD
Uses json.load to fetch the JSON results from the solr API.
url : str
a complete URL that can be passed to urllib.open
request_timeout : int (None)
The number of seconds before the timeout should fail. Leave blank/None
to use the default. __opts__['solr.request_timeout']
Return: dict<str,obj>::
{'success':boolean, 'data':dict, 'errors':list, 'warnings':list}
"""
_auth(url)
try:
request_timeout = __salt__["config.option"]("solr.request_timeout")
if request_timeout is None:
data = json.load(_urlopen(url))
else:
data = json.load(_urlopen(url, timeout=request_timeout))
return _get_return_dict(True, data, [])
except Exception as err:
return _get_return_dict(False, {}, ["{0} : {1}".format(url, err)])
def _http_request(url, request_timeout=None):
'''
PRIVATE METHOD
Uses json.load to fetch the JSON results from the solr API.
url : str
a complete URL that can be passed to urllib.open
request_timeout : int (None)
The number of seconds before the timeout should fail. Leave blank/None
to use the default. __opts__['solr.request_timeout']
Return: dict<str,obj>::
{'success':boolean, 'data':dict, 'errors':list, 'warnings':list}
'''
_auth(url)
try:
request_timeout = __salt__['config.option']('solr.request_timeout')
if request_timeout is None:
data = json.load(_urlopen(url))
else:
data = json.load(_urlopen(url, timeout=request_timeout))
return _get_return_dict(True, data, [])
except Exception as err:
return _get_return_dict(False, {}, ["{0} : {1}".format(url, err)])
def _wget(cmd, opts=None, url='http://localhost:8080/manager', timeout=180):
'''
A private function used to issue the command to tomcat via the manager
webapp
cmd
the command to execute
url
the URL of the server manager webapp
example: http://localhost:8080/manager
opts
a dict of arguments
timeout
timeout for HTTP request
return value is a dict in the from of::
{
res: [True|False]
msg: list of lines we got back from the manager
}
'''
ret = {'res': True, 'msg': []}
# prepare authentication
auth = _auth(url)
if auth is False:
ret['res'] = False
ret['msg'] = 'missing username and password settings (grain/pillar)'
return ret
# prepare URL
if url[-1] != '/':
url += '/'
url6 = url
url += 'text/{0}'.format(cmd)
url6 += '{0}'.format(cmd)
if opts:
url += '?{0}'.format(_urlencode(opts))
url6 += '?{0}'.format(_urlencode(opts))
# Make the HTTP request
_install_opener(auth)
try:
# Trying tomcat >= 7 url
ret['msg'] = _urlopen(url, timeout=timeout).read().splitlines()
except Exception:
try:
# Trying tomcat6 url
ret['msg'] = _urlopen(url6, timeout=timeout).read().splitlines()
except Exception:
ret['msg'] = 'Failed to create HTTP request'
if not ret['msg'][0].startswith('OK'):
ret['res'] = False
return ret
def _do_http(opts, profile="default"):
"""
Make the http request and return the data
"""
ret = {}
url = __salt__["config.get"]("modjk:{0}:url".format(profile), "")
user = __salt__["config.get"]("modjk:{0}:user".format(profile), "")
passwd = __salt__["config.get"]("modjk:{0}:pass".format(profile), "")
realm = __salt__["config.get"]("modjk:{0}:realm".format(profile), "")
timeout = __salt__["config.get"]("modjk:{0}:timeout".format(profile), "")
if not url:
raise Exception("missing url in profile {0}".format(profile))
if user and passwd:
auth = _auth(url=url, realm=realm, user=user, passwd=passwd)
_install_opener(auth)
url += "?{0}".format(_urlencode(opts))
for line in _urlopen(url, timeout=timeout).read().splitlines():
splt = line.split("=", 1)
if splt[0] in ret:
ret[splt[0]] += ",{0}".format(splt[1])
else:
ret[splt[0]] = splt[1]
return ret
def _http_request(url, request_timeout=None):
"""
PRIVATE METHOD
Uses salt.utils.json.load to fetch the JSON results from the solr API.
url : str
a complete URL that can be passed to urllib.open
request_timeout : int (None)
The number of seconds before the timeout should fail. Leave blank/None
to use the default. __opts__['solr.request_timeout']
Return: dict<str,obj>::
{'success':boolean, 'data':dict, 'errors':list, 'warnings':list}
"""
_auth(url)
try:
request_timeout = __salt__["config.option"]("solr.request_timeout")
kwargs = {} if request_timeout is None else {
"timeout": request_timeout
}
data = salt.utils.json.load(_urlopen(url, **kwargs))
return _get_return_dict(True, data, [])
except Exception as err: # pylint: disable=broad-except
return _get_return_dict(False, {}, ["{0} : {1}".format(url, err)])
def _get_all_packages(mirror=DEFAULT_MIRROR, cyg_arch="x86_64"):
"""
Return the list of packages based on the mirror provided.
"""
if "cyg.all_packages" not in __context__:
__context__["cyg.all_packages"] = {}
if mirror not in __context__["cyg.all_packages"]:
__context__["cyg.all_packages"][mirror] = []
if not __context__["cyg.all_packages"][mirror]:
pkg_source = "/".join([mirror, cyg_arch, "setup.bz2"])
file_data = _urlopen(pkg_source).read()
file_lines = (
bz2.decompress(file_data).decode("utf_8", errors="replace").splitlines()
)
packages = [
re.search("^@ ([^ ]+)", line).group(1)
for line in file_lines
if re.match("^@ [^ ]+", line)
]
__context__["cyg.all_packages"][mirror] = packages
return __context__["cyg.all_packages"][mirror]
def _do_http(opts, profile='default'):
'''
Make the http request and return the data
'''
ret = {}
url = __salt__['config.get']('modjk:{0}:url'.format(profile), '')
user = __salt__['config.get']('modjk:{0}:user'.format(profile), '')
passwd = __salt__['config.get']('modjk:{0}:pass'.format(profile), '')
realm = __salt__['config.get']('modjk:{0}:realm'.format(profile), '')
timeout = __salt__['config.get']('modjk:{0}:timeout'.format(profile), '')
if not url:
raise Exception('missing url in profile {0}'.format(profile))
if user and passwd:
auth = _auth(url=url, realm=realm, user=user, passwd=passwd)
_install_opener(auth)
url += '?{0}'.format(_urlencode(opts))
for line in _urlopen(url, timeout=timeout).read().splitlines():
splt = line.split('=', 1)
if splt[0] in ret:
ret[splt[0]] += ',{0}'.format(splt[1])
else:
ret[splt[0]] = splt[1]
return ret
def _do_http(opts, profile='default'):
'''
Make the http request and return the data
'''
ret = {}
url = __salt__['config.get']('modjk:{0}:url'.format(profile), '')
user = __salt__['config.get']('modjk:{0}:user'.format(profile), '')
passwd = __salt__['config.get']('modjk:{0}:pass'.format(profile), '')
realm = __salt__['config.get']('modjk:{0}:realm'.format(profile), '')
timeout = __salt__['config.get']('modjk:{0}:timeout'.format(profile), '')
if not url:
raise Exception('missing url in profile {0}'.format(profile))
if user and passwd:
auth = _auth(url=url, realm=realm, user=user, passwd=passwd)
_install_opener(auth)
url += '?{0}'.format(_urlencode(opts))
for line in _urlopen(url, timeout=timeout).read().splitlines():
splt = line.split('=', 1)
if splt[0] in ret:
ret[splt[0]] += ',{0}'.format(splt[1])
else:
ret[splt[0]] = splt[1]
return ret
def _run_silent_cygwin(cyg_arch='x86_64',
args=None,
mirrors=None):
"""
Retrieve the correct setup.exe.
Run it with the correct arguments to get the bare minimum cygwin
installation up and running.
"""
cyg_cache_dir = os.sep.join(['c:', 'cygcache'])
cyg_setup = 'setup-{0}.exe'.format(cyg_arch)
cyg_setup_path = os.sep.join([cyg_cache_dir, cyg_setup])
cyg_setup_source = 'http://cygwin.com/{0}'.format(cyg_setup)
# cyg_setup_source_hash = 'http://cygwin.com/{0}.sig'.format(cyg_setup)
# until a hash gets published that we can verify the newest setup against
# just go ahead and download a new one.
if not os.path.exists(cyg_cache_dir):
os.mkdir(cyg_cache_dir)
elif os.path.exists(cyg_setup_path):
os.remove(cyg_setup_path)
file_data = _urlopen(cyg_setup_source)
with salt.utils.files.fopen(cyg_setup_path, "wb") as fhw:
fhw.write(file_data.read())
setup_command = cyg_setup_path
options = []
options.append('--local-package-dir {0}'.format(cyg_cache_dir))
if mirrors is None:
mirrors = [{DEFAULT_MIRROR: DEFAULT_MIRROR_KEY}]
for mirror in mirrors:
for mirror_url, key in mirror.items():
options.append('--site {0}'.format(mirror_url))
if key:
options.append('--pubkey {0}'.format(key))
options.append('--no-desktop')
options.append('--quiet-mode')
options.append('--disable-buggy-antivirus')
if args is not None:
for arg in args:
options.append(arg)
cmdline_args = ' '.join(options)
setup_command = ' '.join([cyg_setup_path, cmdline_args])
ret = __salt__['cmd.run_all'](
setup_command
)
if ret['retcode'] == 0:
return ret['stdout']
else:
return False
def _run_silent_cygwin(cyg_arch='x86_64',
args=None,
mirrors=None):
"""
Retrieve the correct setup.exe.
Run it with the correct arguments to get the bare minimum cygwin
installation up and running.
"""
cyg_cache_dir = os.sep.join(['c:', 'cygcache'])
cyg_setup = 'setup-{0}.exe'.format(cyg_arch)
cyg_setup_path = os.sep.join([cyg_cache_dir, cyg_setup])
cyg_setup_source = 'http://cygwin.com/{0}'.format(cyg_setup)
# cyg_setup_source_hash = 'http://cygwin.com/{0}.sig'.format(cyg_setup)
# until a hash gets published that we can verify the newest setup against
# just go ahead and download a new one.
if not os.path.exists(cyg_cache_dir):
os.mkdir(cyg_cache_dir)
elif os.path.exists(cyg_setup_path):
os.remove(cyg_setup_path)
file_data = _urlopen(cyg_setup_source)
with salt.utils.fopen(cyg_setup_path, "wb") as fhw:
fhw.write(file_data.read())
setup_command = cyg_setup_path
options = []
options.append('--local-package-dir {0}'.format(cyg_cache_dir))
if mirrors is None:
mirrors = [{DEFAULT_MIRROR: DEFAULT_MIRROR_KEY}]
for mirror in mirrors:
for mirror_url, key in mirror.items():
options.append('--site {0}'.format(mirror_url))
if key:
options.append('--pubkey {0}'.format(key))
options.append('--no-desktop')
options.append('--quiet-mode')
options.append('--disable-buggy-antivirus')
if args is not None:
for arg in args:
options.append(arg)
cmdline_args = ' '.join(options)
setup_command = ' '.join([cyg_setup_path, cmdline_args])
ret = __salt__['cmd.run_all'](
setup_command
)
if ret['retcode'] == 0:
return ret['stdout']
else:
return False
def _check_cygwin_installed(cyg_arch="x86_64"):
"""
Return True or False if cygwin is installed.
Use the cygcheck executable to check install. It is installed as part of
the base package, and we use it to check packages
'''
path_to_cygcheck = os.sep.join(['C:',
_get_cyg_dir(cyg_arch),
'bin', 'cygcheck.exe'])
LOG.debug('Path to cygcheck.exe: %s', path_to_cygcheck)
if not os.path.exists(path_to_cygcheck):
LOG.debug("Could not find cygcheck.exe")
return False
return True
def _get_all_packages(mirror=DEFAULT_MIRROR, cyg_arch="x86_64"):
"""
Return the list of packages based on the mirror provided.
'''
if 'cyg.all_packages' not in __context__:
__context__['cyg.all_packages'] = {}
if mirror not in __context__['cyg.all_packages']:
__context__['cyg.all_packages'][mirror] = []
if not __context__['cyg.all_packages'][mirror]:
pkg_source = '/'.join([mirror, cyg_arch, 'setup.bz2'])
file_data = _urlopen(pkg_source).read()
file_lines = (
bz2.decompress(file_data).decode("utf_8", errors="replace").splitlines()
)
packages = [
re.search("^@ ([^ ]+)", line).group(1)
for line in file_lines
if re.match("^@ [^ ]+", line)
]
__context__["cyg.all_packages"][mirror] = packages
return __context__["cyg.all_packages"][mirror]
def external_ip():
'''
Return the external IP address
'''
check_ips = ('http://ipecho.net/plain',
'http://api.externalip.net/ip',
'http://v4.ident.me')
for url in check_ips:
try:
with contextlib.closing(_urlopen(url, timeout=3)) as req:
ip_ = req.read().strip()
if not _ipv4_addr(ip_):
continue
return {'external_ip': ip_}
except (HTTPError, URLError, socket.timeout):
continue
# Return an empty value as a last resort
return {'external_ip': []}
def _get_all_packages(mirror=DEFAULT_MIRROR,
cyg_arch='x86_64'):
"""Return the list of packages based on the mirror provided."""
if 'cyg.all_packages' not in __context__:
__context__['cyg.all_packages'] = {}
if mirror not in __context__['cyg.all_packages']:
__context__['cyg.all_packages'][mirror] = []
if not len(__context__['cyg.all_packages'][mirror]):
pkg_source = '/'.join([mirror, cyg_arch, 'setup.bz2'])
file_data = _urlopen(pkg_source).read()
file_lines = bz2.decompress(file_data).decode('utf_8',
errors='replace'
).splitlines()
packages = [re.search('^@ ([^ ]+)', line).group(1) for
line in file_lines if re.match('^@ [^ ]+', line)]
__context__['cyg.all_packages'][mirror] = packages
return __context__['cyg.all_packages'][mirror]
def _get_all_packages(mirror=DEFAULT_MIRROR,
cyg_arch='x86_64'):
"""Return the list of packages based on the mirror provided."""
if 'cyg.all_packages' not in __context__:
__context__['cyg.all_packages'] = {}
if mirror not in __context__['cyg.all_packages']:
__context__['cyg.all_packages'][mirror] = []
if not len(__context__['cyg.all_packages'][mirror]):
pkg_source = '/'.join([mirror, cyg_arch, 'setup.bz2'])
file_data = _urlopen(pkg_source).read()
file_lines = bz2.decompress(file_data).decode('utf_8',
errors='replace'
).splitlines()
packages = [re.search('^@ ([^ ]+)', line).group(1) for
line in file_lines if re.match('^@ [^ ]+', line)]
__context__['cyg.all_packages'][mirror] = packages
return __context__['cyg.all_packages'][mirror]
def status(url="http://127.0.0.1/status"):
"""
Return the data from an Nginx status page as a dictionary.
http://wiki.nginx.org/HttpStubStatusModule
url
The URL of the status page. Defaults to 'http://127.0.0.1/status'
CLI Example:
.. code-block:: bash
salt '*' nginx.status
"""
resp = _urlopen(url)
status_data = resp.read()
resp.close()
lines = status_data.splitlines()
if not len(lines) == 4:
return
# "Active connections: 1 "
active_connections = lines[0].split()[2]
# "server accepts handled requests"
# " 12 12 9 "
accepted, handled, requests = lines[2].split()
# "Reading: 0 Writing: 1 Waiting: 0 "
_, reading, _, writing, _, waiting = lines[3].split()
return {
'active connections': int(active_connections),
'accepted': int(accepted),
'handled': int(handled),
'requests': int(requests),
'reading': int(reading),
'writing': int(writing),
'waiting': int(waiting),
}
def status(url="http://127.0.0.1/status"):
"""
Return the data from an Nginx status page as a dictionary.
http://wiki.nginx.org/HttpStubStatusModule
url
The URL of the status page. Defaults to 'http://127.0.0.1/status'
CLI Example:
.. code-block:: bash
salt '*' nginx.status
"""
resp = _urlopen(url)
status_data = resp.read()
resp.close()
lines = status_data.splitlines()
if not len(lines) == 4:
return
# "Active connections: 1 "
active_connections = lines[0].split()[2]
# "server accepts handled requests"
# " 12 12 9 "
accepted, handled, requests = lines[2].split()
# "Reading: 0 Writing: 1 Waiting: 0 "
_, reading, _, writing, _, waiting = lines[3].split()
return {
'active connections': int(active_connections),
'accepted': int(accepted),
'handled': int(handled),
'requests': int(requests),
'reading': int(reading),
'writing': int(writing),
'waiting': int(waiting),
}
def upgrade_bootstrap(directory='.',
onlyif=None,
unless=None,
runas=None,
env=(),
offline=False,
buildout_ver=None):
'''
Upgrade current bootstrap.py with the last released one.
Indeed, when we first run a buildout, a common source of problem
is to have a locally stale bootstrap, we just try to grab a new copy
directory
directory to execute in
offline
are we executing buildout in offline mode
buildout_ver
forcing to use a specific buildout version (1 | 2)
onlyif
Only execute cmd if statement on the host return 0
unless
Do not execute cmd if statement on the host return 0
CLI Example:
.. code-block:: bash
salt '*' buildout.upgrade_bootstrap /srv/mybuildout
'''
if buildout_ver:
booturl = _URL_VERSIONS[buildout_ver]
else:
buildout_ver = _get_buildout_ver(directory)
booturl = _get_bootstrap_url(directory)
LOG.debug('Using {0}'.format(booturl))
# try to download an up-to-date bootstrap
# set defaulttimeout
# and add possible content
directory = os.path.abspath(directory)
b_py = os.path.join(directory, 'bootstrap.py')
comment = ''
try:
oldcontent = _get_bootstrap_content(directory)
dbuild = _dot_buildout(directory)
data = oldcontent
updated = False
dled = False
if not offline:
try:
if not os.path.isdir(dbuild):
os.makedirs(dbuild)
# only try to download once per buildout checkout
with salt.utils.fopen(
os.path.join(
dbuild,
'{0}.updated_bootstrap'.format(buildout_ver))):
pass
except (OSError, IOError):
LOG.info('Bootstrap updated from repository')
data = _urlopen(booturl).read()
updated = True
dled = True
if 'socket.setdefaulttimeout' not in data:
updated = True
ldata = data.splitlines()
ldata.insert(1, 'import socket;socket.setdefaulttimeout(2)')
data = '\n'.join(ldata)
if updated:
comment = 'Bootstrap updated'
with salt.utils.fopen(b_py, 'w') as fic:
fic.write(data)
if dled:
with salt.utils.fopen(
os.path.join(dbuild,
'{0}.updated_bootstrap'.format(buildout_ver)),
'w') as afic:
afic.write('foo')
except (OSError, IOError):
if oldcontent:
with salt.utils.fopen(b_py, 'w') as fic:
fic.write(oldcontent)
return {'comment': comment}
def server_status(profile="default"):
"""
Get Information from the Apache server-status handler
.. note::
The server-status handler is disabled by default.
In order for this function to work it needs to be enabled.
See http://httpd.apache.org/docs/2.2/mod/mod_status.html
The following configuration needs to exists in pillar/grains.
Each entry nested in ``apache.server-status`` is a profile of a vhost/server.
This would give support for multiple apache servers/vhosts.
.. code-block:: yaml
apache.server-status:
default:
url: http://localhost/server-status
user: someuser
pass: password
realm: 'authentication realm for digest passwords'
timeout: 5
CLI Examples:
.. code-block:: bash
salt '*' apache.server_status
salt '*' apache.server_status other-profile
"""
ret = {
"Scoreboard": {
"_": 0,
"S": 0,
"R": 0,
"W": 0,
"K": 0,
"D": 0,
"C": 0,
"L": 0,
"G": 0,
"I": 0,
".": 0,
},
}
# Get configuration from pillar
url = __salt__["config.get"](
"apache.server-status:{0}:url".format(profile),
"http://localhost/server-status")
user = __salt__["config.get"](
"apache.server-status:{0}:user".format(profile), "")
passwd = __salt__["config.get"](
"apache.server-status:{0}:pass".format(profile), "")
realm = __salt__["config.get"](
"apache.server-status:{0}:realm".format(profile), "")
timeout = __salt__["config.get"](
"apache.server-status:{0}:timeout".format(profile), 5)
# create authentication handler if configuration exists
if user and passwd:
basic = _HTTPBasicAuthHandler()
basic.add_password(realm=realm, uri=url, user=user, passwd=passwd)
digest = _HTTPDigestAuthHandler()
digest.add_password(realm=realm, uri=url, user=user, passwd=passwd)
_install_opener(_build_opener(basic, digest))
# get http data
url += "?auto"
try:
response = _urlopen(url, timeout=timeout).read().splitlines()
except URLError:
return "error"
# parse the data
for line in response:
splt = line.split(":", 1)
splt[0] = splt[0].strip()
splt[1] = splt[1].strip()
if splt[0] == "Scoreboard":
for c in splt[1]:
ret["Scoreboard"][c] += 1
else:
if splt[1].isdigit():
ret[splt[0]] = int(splt[1])
else:
ret[splt[0]] = float(splt[1])
# return the good stuff
return ret
def status(url="http://127.0.0.1/status"):
"""
Return the data from an Nginx status page as a dictionary.
http://wiki.nginx.org/HttpStubStatusModule
url
The URL of the status page. Defaults to 'http://127.0.0.1/status'
CLI Example:
.. code-block:: bash
salt '*' nginx.status
"""
resp = _urlopen(url)
status_data = resp.read()
resp.close()
lines = status_data.splitlines()
if not len(lines) == 4:
return
# "Active connections: 1 "
active_connections = lines[0].split()[2]
# "server accepts handled requests"
# " 12 12 9 "
accepted, handled, requests = lines[2].split()
# "Reading: 0 Writing: 1 Waiting: 0 "
_, reading, _, writing, _, waiting = lines[3].split()
return {
"active connections": int(active_connections),
def server_status(profile='default'):
'''
Get Information from the Apache server-status handler
.. note::
The server-status handler is disabled by default.
In order for this function to work it needs to be enabled.
See http://httpd.apache.org/docs/2.2/mod/mod_status.html
The following configuration needs to exists in pillar/grains.
Each entry nested in ``apache.server-status`` is a profile of a vhost/server.
This would give support for multiple apache servers/vhosts.
.. code-block:: yaml
apache.server-status:
default:
url: http://localhost/server-status
user: someuser
pass: password
realm: 'authentication realm for digest passwords'
timeout: 5
CLI Examples:
.. code-block:: bash
salt '*' apache.server_status
salt '*' apache.server_status other-profile
'''
ret = {
'Scoreboard': {
'_': 0,
'S': 0,
'R': 0,
'W': 0,
'K': 0,
'D': 0,
'C': 0,
'L': 0,
'G': 0,
'I': 0,
'.': 0,
},
}
# Get configuration from pillar
url = __salt__['config.get'](
'apache.server-status:{0}:url'.format(profile),
'http://localhost/server-status'
)
user = __salt__['config.get'](
'apache.server-status:{0}:user'.format(profile),
''
)
passwd = __salt__['config.get'](
'apache.server-status:{0}:pass'.format(profile),
''
)
realm = __salt__['config.get'](
'apache.server-status:{0}:realm'.format(profile),
''
)
timeout = __salt__['config.get'](
'apache.server-status:{0}:timeout'.format(profile),
5
)
# create authentication handler if configuration exists
if user and passwd:
basic = _HTTPBasicAuthHandler()
basic.add_password(realm=realm, uri=url, user=user, passwd=passwd)
digest = _HTTPDigestAuthHandler()
digest.add_password(realm=realm, uri=url, user=user, passwd=passwd)
_install_opener(_build_opener(basic, digest))
# get http data
url += '?auto'
try:
response = _urlopen(url, timeout=timeout).read().splitlines()
except URLError:
return 'error'
# parse the data
for line in response:
splt = line.split(':', 1)
splt[0] = splt[0].strip()
splt[1] = splt[1].strip()
if splt[0] == 'Scoreboard':
for c in splt[1]:
ret['Scoreboard'][c] += 1
else:
if splt[1].isdigit():
ret[splt[0]] = int(splt[1])
else:
ret[splt[0]] = float(splt[1])
# return the good stuff
return ret
def server_status(profile='default'):
'''
Get Information from the Apache server-status handler
.. note::
The server-status handler is disabled by default.
In order for this function to work it needs to be enabled.
See http://httpd.apache.org/docs/2.2/mod/mod_status.html
The following configuration needs to exists in pillar/grains.
Each entry nested in ``apache.server-status`` is a profile of a vhost/server.
This would give support for multiple apache servers/vhosts.
.. code-block:: yaml
apache.server-status:
default:
url: http://localhost/server-status
user: someuser
pass: password
realm: 'authentication realm for digest passwords'
timeout: 5
CLI Examples:
.. code-block:: bash
salt '*' apache.server_status
salt '*' apache.server_status other-profile
'''
ret = {
'Scoreboard': {
'_': 0,
'S': 0,
'R': 0,
'W': 0,
'K': 0,
'D': 0,
'C': 0,
'L': 0,
'G': 0,
'I': 0,
'.': 0,
},
}
# Get configuration from pillar
url = __salt__['config.get'](
'apache.server-status:{0}:url'.format(profile),
'http://localhost/server-status'
)
user = __salt__['config.get'](
'apache.server-status:{0}:user'.format(profile),
''
)
passwd = __salt__['config.get'](
'apache.server-status:{0}:pass'.format(profile),
''
)
realm = __salt__['config.get'](
'apache.server-status:{0}:realm'.format(profile),
''
)
timeout = __salt__['config.get'](
'apache.server-status:{0}:timeout'.format(profile),
5
)
# create authentication handler if configuration exists
if user and passwd:
basic = _HTTPBasicAuthHandler()
basic.add_password(realm=realm, uri=url, user=user, passwd=passwd)
digest = _HTTPDigestAuthHandler()
digest.add_password(realm=realm, uri=url, user=user, passwd=passwd)
_install_opener(_build_opener(basic, digest))
# get http data
url += '?auto'
try:
response = _urlopen(url, timeout=timeout).read().splitlines()
except URLError:
return 'error'
# parse the data
for line in response:
splt = line.split(':', 1)
splt[0] = splt[0].strip()
splt[1] = splt[1].strip()
if splt[0] == 'Scoreboard':
for c in splt[1]:
ret['Scoreboard'][c] += 1
else:
if splt[1].isdigit():
ret[splt[0]] = int(splt[1])
else:
ret[splt[0]] = float(splt[1])
# return the good stuff
return ret
def query(action=None, command=None, args=None, method="GET", data=None):
"""
Make a web call to a Parallels provider
"""
path = config.get_cloud_config_value(
"url", get_configured_provider(), __opts__, search_global=False
)
auth_handler = _HTTPBasicAuthHandler()
auth_handler.add_password(
realm="Parallels Instance Manager",
uri=path,
user=config.get_cloud_config_value(
"user", get_configured_provider(), __opts__, search_global=False
),
passwd=config.get_cloud_config_value(
"password", get_configured_provider(), __opts__, search_global=False
),
)
opener = _build_opener(auth_handler)
_install_opener(opener)
if action:
path += action
if command:
path += "/{0}".format(command)
if not type(args, dict):
args = {}
kwargs = {"data": data}
if isinstance(data, six.string_types) and "<?xml" in data:
kwargs["headers"] = {
"Content-type": "application/xml",
}
if args:
params = _urlencode(args)
req = _Request(url="{0}?{1}".format(path, params), **kwargs)
else:
req = _Request(url=path, **kwargs)
req.get_method = lambda: method
log.debug("%s %s", method, req.get_full_url())
if data:
log.debug(data)
try:
result = _urlopen(req)
log.debug("PARALLELS Response Status Code: %s", result.getcode())
if "content-length" in result.headers:
content = result.read()
result.close()
items = ET.fromstring(content)
return items
return {}
except URLError as exc:
log.error("PARALLELS Response Status Code: %s %s", exc.code, exc.msg)
root = ET.fromstring(exc.read())
log.error(root)
return {"error": root}
def query(action=None, command=None, args=None, method='GET', data=None):
'''
Make a web call to a Parallels provider
'''
path = config.get_cloud_config_value('url',
get_configured_provider(),
__opts__,
search_global=False)
auth_handler = _HTTPBasicAuthHandler()
auth_handler.add_password(
realm='Parallels Instance Manager',
uri=path,
user=config.get_cloud_config_value('user',
get_configured_provider(),
__opts__,
search_global=False),
passwd=config.get_cloud_config_value('password',
get_configured_provider(),
__opts__,
search_global=False))
opener = _build_opener(auth_handler)
_install_opener(opener)
if action:
path += action
if command:
path += '/{0}'.format(command)
if not type(args, dict):
args = {}
kwargs = {'data': data}
if isinstance(data, six.string_types) and '<?xml' in data:
kwargs['headers'] = {
'Content-type': 'application/xml',
}
if args:
params = _urlencode(args)
req = _Request(url='{0}?{1}'.format(path, params), **kwargs)
else:
req = _Request(url=path, **kwargs)
req.get_method = lambda: method
log.debug('{0} {1}'.format(method, req.get_full_url()))
if data:
log.debug(data)
try:
result = _urlopen(req)
log.debug('PARALLELS Response Status Code: {0}'.format(
result.getcode()))
if 'content-length' in result.headers:
content = result.read()
result.close()
items = ET.fromstring(content)
return items
return {}
except URLError as exc:
log.error('PARALLELS Response Status Code: {0} {1}'.format(
exc.code, exc.msg))
root = ET.fromstring(exc.read())
log.error(root)
return {'error': root}
def bootstrap_psexec(
hosts="", master=None, version=None, arch="win32", installer_url=None, username=None, password=None
):
"""
Bootstrap Windows minions via PsExec.
hosts
Comma separated list of hosts to deploy the Windows Salt minion.
master
Address of the Salt master passed as an argument to the installer.
version
Point release of installer to download. Defaults to the most recent.
arch
Architecture of installer to download. Defaults to win32.
installer_url
URL of minion installer executable. Defaults to the latest version from
http://docs.saltstack.com/downloads
username
Optional user name for login on remote computer.
password
Password for optional username. If omitted, PsExec will prompt for one
to be entered for each host.
CLI Example:
.. code-block:: bash
salt-run manage.bootstrap_psexec hosts='host1,host2'
salt-run manage.bootstrap_psexec hosts='host1,host2' version='0.17' username='******'
salt-run manage.bootstrap_psexec hosts='host1,host2' installer_url='http://exampledomain/salt-installer.exe'
"""
if not installer_url:
base_url = "http://docs.saltstack.com/downloads/"
source = _urlopen(base_url).read()
salty_rx = re.compile('>(Salt-Minion-(.+?)-(.+)-Setup.exe)</a></td><td align="right">(.*?)\\s*<')
source_list = sorted(
[
[path, ver, plat, time.strptime(date, "%d-%b-%Y %H:%M")]
for path, ver, plat, date in salty_rx.findall(source)
],
key=operator.itemgetter(3),
reverse=True,
)
if version:
source_list = [s for s in source_list if s[1] == version]
if arch:
source_list = [s for s in source_list if s[2] == arch]
if not source_list:
return -1
version = source_list[0][1]
arch = source_list[0][2]
installer_url = base_url + source_list[0][0]
# It's no secret that Windows is notoriously command-line hostile.
# Win 7 and newer can use PowerShell out of the box, but to reach
# all those XP and 2K3 machines we must suppress our gag-reflex
# and use VB!
# The following script was borrowed from an informative article about
# downloading exploit payloads for malware. Nope, no irony here.
# http://www.greyhathacker.net/?p=500
vb_script = """strFileURL = "{0}"
strHDLocation = "{1}"
Set objXMLHTTP = CreateObject("MSXML2.XMLHTTP")
objXMLHTTP.open "GET", strFileURL, false
objXMLHTTP.send()
If objXMLHTTP.Status = 200 Then
Set objADOStream = CreateObject("ADODB.Stream")
objADOStream.Open
objADOStream.Type = 1
objADOStream.Write objXMLHTTP.ResponseBody
objADOStream.Position = 0
objADOStream.SaveToFile strHDLocation
objADOStream.Close
Set objADOStream = Nothing
End if
Set objXMLHTTP = Nothing
Set objShell = CreateObject("WScript.Shell")
objShell.Exec("{1}{2}")"""
vb_saltexec = "saltinstall.exe"
vb_saltexec_args = " /S /minion-name=%COMPUTERNAME%"
if master:
vb_saltexec_args += " /master={0}".format(master)
# One further thing we need to do; the Windows Salt minion is pretty
# self-contained, except for the Microsoft Visual C++ 2008 runtime.
# It's tiny, so the bootstrap will attempt a silent install.
vb_vcrunexec = "vcredist.exe"
if arch == "AMD64":
vb_vcrun = vb_script.format(
"http://download.microsoft.com/download/d/2/4/d242c3fb-da5a-4542-ad66-f9661d0a8d19/vcredist_x64.exe",
vb_vcrunexec,
" /q",
)
else:
vb_vcrun = vb_script.format(
"http://download.microsoft.com/download/d/d/9/dd9a82d0-52ef-40db-8dab-795376989c03/vcredist_x86.exe",
vb_vcrunexec,
" /q",
)
vb_salt = vb_script.format(installer_url, vb_saltexec, vb_saltexec_args)
# PsExec doesn't like extra long arguments; save the instructions as a batch
# file so we can fire it over for execution.
# First off, change to the local temp directory, stop salt-minion (if
# running), and remove the master's public key.
# This is to accommodate for reinstalling Salt over an old or broken build,
# e.g. if the master address is changed, the salt-minion process will fail
# to authenticate and quit; which means infinite restarts under Windows.
batch = "cd /d %TEMP%\nnet stop salt-minion\ndel c:\\salt\\conf\\pki\\minion\\minion_master.pub\n"
# Speaking of command-line hostile, cscript only supports reading a script
# from a file. Glue it together line by line.
for x, y in ((vb_vcrunexec, vb_vcrun), (vb_saltexec, vb_salt)):
vb_lines = y.split("\n")
batch += (
"\ndel "
+ x
+ "\n@echo "
+ vb_lines[0]
+ " >"
+ x
+ ".vbs\n@echo "
+ (" >>" + x + ".vbs\n@echo ").join(vb_lines[1:])
+ " >>"
+ x
+ ".vbs\ncscript.exe /NoLogo "
+ x
+ ".vbs"
)
batch_path = tempfile.mkstemp(suffix=".bat")[1]
with salt.utils.fopen(batch_path, "wb") as batch_file:
batch_file.write(batch)
for host in hosts.split(","):
argv = ["psexec", "\\\\" + host]
if username:
argv += ["-u", username]
if password:
argv += ["-p", password]
argv += ["-h", "-c", batch_path]
subprocess.call(argv)
def _wget(cmd, opts=None, url="http://localhost:8080/manager", timeout=180):
"""
A private function used to issue the command to tomcat via the manager
webapp
cmd
the command to execute
url
The URL of the server manager webapp (example:
http://localhost:8080/manager)
opts
a dict of arguments
timeout
timeout for HTTP request
Return value is a dict in the from of::
{
res: [True|False]
msg: list of lines we got back from the manager
}
"""
ret = {"res": True, "msg": []}
# prepare authentication
auth = _auth(url)
if auth is False:
ret["res"] = False
ret["msg"] = "missing username and password settings (grain/pillar)"
return ret
# prepare URL
if url[-1] != "/":
url += "/"
url6 = url
url += "text/{0}".format(cmd)
url6 += "{0}".format(cmd)
if opts:
url += "?{0}".format(_urlencode(opts))
url6 += "?{0}".format(_urlencode(opts))
# Make the HTTP request
_install_opener(auth)
try:
# Trying tomcat >= 7 url
ret["msg"] = _urlopen(url, timeout=timeout).read().splitlines()
except Exception: # pylint: disable=broad-except
try:
# Trying tomcat6 url
ret["msg"] = _urlopen(url6, timeout=timeout).read().splitlines()
except Exception: # pylint: disable=broad-except
ret["msg"] = "Failed to create HTTP request"
# Force all byte strings to utf-8 strings, for python >= 3.4
for key, value in enumerate(ret["msg"]):
try:
ret["msg"][key] = salt.utils.stringutils.to_unicode(value, "utf-8")
except (UnicodeDecodeError, AttributeError):
pass
if not ret["msg"][0].startswith("OK"):
ret["res"] = False
return ret
def upgrade_bootstrap(
directory=".",
onlyif=None,
unless=None,
runas=None,
env=(),
offline=False,
buildout_ver=None,
):
"""
Upgrade current bootstrap.py with the last released one.
Indeed, when we first run a buildout, a common source of problem
is to have a locally stale bootstrap, we just try to grab a new copy
directory
directory to execute in
offline
are we executing buildout in offline mode
buildout_ver
forcing to use a specific buildout version (1 | 2)
onlyif
Only execute cmd if statement on the host return 0
unless
Do not execute cmd if statement on the host return 0
CLI Example:
.. code-block:: bash
salt '*' buildout.upgrade_bootstrap /srv/mybuildout
"""
if buildout_ver:
booturl = _URL_VERSIONS[buildout_ver]
else:
buildout_ver = _get_buildout_ver(directory)
booturl = _get_bootstrap_url(directory)
LOG.debug("Using {0}".format(booturl)) # pylint: disable=str-format-in-logging
# try to download an up-to-date bootstrap
# set defaulttimeout
# and add possible content
directory = os.path.abspath(directory)
b_py = os.path.join(directory, "bootstrap.py")
comment = ""
try:
oldcontent = _get_bootstrap_content(directory)
dbuild = _dot_buildout(directory)
data = oldcontent
updated = False
dled = False
if not offline:
try:
if not os.path.isdir(dbuild):
os.makedirs(dbuild)
# only try to download once per buildout checkout
with salt.utils.files.fopen(
os.path.join(
dbuild,
"{0}.updated_bootstrap".format(buildout_ver))):
pass
except (OSError, IOError):
LOG.info("Bootstrap updated from repository")
data = _urlopen(booturl).read()
updated = True
dled = True
if "socket.setdefaulttimeout" not in data:
updated = True
ldata = data.splitlines()
ldata.insert(1, "import socket;socket.setdefaulttimeout(2)")
data = "\n".join(ldata)
if updated:
comment = "Bootstrap updated"
with salt.utils.files.fopen(b_py, "w") as fic:
fic.write(salt.utils.stringutils.to_str(data))
if dled:
with salt.utils.files.fopen(
os.path.join(dbuild,
"{0}.updated_bootstrap".format(buildout_ver)),
"w") as afic:
afic.write("foo")
except (OSError, IOError):
if oldcontent:
with salt.utils.files.fopen(b_py, "w") as fic:
fic.write(salt.utils.stringutils.to_str(oldcontent))
return {"comment": comment}
def query(action=None, command=None, args=None, method='GET', data=None):
'''
Make a web call to a Parallels provider
'''
path = config.get_cloud_config_value(
'url', get_configured_provider(), __opts__, search_global=False
)
auth_handler = _HTTPBasicAuthHandler()
auth_handler.add_password(
realm='Parallels Instance Manager',
uri=path,
user=config.get_cloud_config_value(
'user', get_configured_provider(), __opts__, search_global=False
),
passwd=config.get_cloud_config_value(
'password', get_configured_provider(), __opts__,
search_global=False
)
)
opener = _build_opener(auth_handler)
_install_opener(opener)
if action:
path += action
if command:
path += '/{0}'.format(command)
if not type(args, dict):
args = {}
kwargs = {'data': data}
if isinstance(data, str) and '<?xml' in data:
kwargs['headers'] = {
'Content-type': 'application/xml',
}
if args:
params = _urlencode(args)
req = _Request(url='{0}?{1}'.format(path, params), **kwargs)
else:
req = _Request(url=path, **kwargs)
req.get_method = lambda: method
log.debug('{0} {1}'.format(method, req.get_full_url()))
if data:
log.debug(data)
try:
result = _urlopen(req)
log.debug(
'PARALLELS Response Status Code: {0}'.format(
result.getcode()
)
)
if 'content-length' in result.headers:
content = result.read()
result.close()
items = ET.fromstring(content)
return items
return {}
except URLError as exc:
log.error(
'PARALLELS Response Status Code: {0} {1}'.format(
exc.code,
exc.msg
)
)
root = ET.fromstring(exc.read())
log.error(root)
return {'error': root}
def _wget(cmd, opts=None, url='http://localhost:8080/manager', timeout=180):
'''
A private function used to issue the command to tomcat via the manager
webapp
cmd
the command to execute
url
the URL of the server manager webapp
example: http://localhost:8080/manager
opts
a dict of arguments
timeout
timeout for HTTP request
return value is a dict in the from of::
{
res: [True|False]
msg: list of lines we got back from the manager
}
'''
ret = {
'res': True,
'msg': []
}
# prepare authentication
auth = _auth(url)
if auth is False:
ret['res'] = False
ret['msg'] = 'missing username and password settings (grain/pillar)'
return ret
# prepare URL
if url[-1] != '/':
url += '/'
url6 = url
url += 'text/{0}'.format(cmd)
url6 += '{0}'.format(cmd)
if opts:
url += '?{0}'.format(_urlencode(opts))
url6 += '?{0}'.format(_urlencode(opts))
# Make the HTTP request
_install_opener(auth)
try:
# Trying tomcat >= 7 url
ret['msg'] = _urlopen(url, timeout=timeout).read().splitlines()
except Exception:
try:
# Trying tomcat6 url
ret['msg'] = _urlopen(url6, timeout=timeout).read().splitlines()
except Exception:
ret['msg'] = 'Failed to create HTTP request'
if not ret['msg'][0].startswith('OK'):
ret['res'] = False
return ret
def bootstrap_psexec(hosts='', master=None, version=None, arch='win32',
installer_url=None, username=None, password=None):
'''
Bootstrap Windows minions via PsExec.
hosts
Comma separated list of hosts to deploy the Windows Salt minion.
master
Address of the Salt master passed as an argument to the installer.
version
Point release of installer to download. Defaults to the most recent.
arch
Architecture of installer to download. Defaults to win32.
installer_url
URL of minion installer executable. Defaults to the latest version from
https://repo.saltstack.com/windows/
username
Optional user name for login on remote computer.
password
Password for optional username. If omitted, PsExec will prompt for one
to be entered for each host.
CLI Example:
.. code-block:: bash
salt-run manage.bootstrap_psexec hosts='host1,host2'
salt-run manage.bootstrap_psexec hosts='host1,host2' version='0.17' username='******'
salt-run manage.bootstrap_psexec hosts='host1,host2' installer_url='http://exampledomain/salt-installer.exe'
'''
if not installer_url:
base_url = 'https://repo.saltstack.com/windows/'
source = _urlopen(base_url).read()
salty_rx = re.compile('>(Salt-Minion-(.+?)-(.+)-Setup.exe)</a></td><td align="right">(.*?)\\s*<')
source_list = sorted([[path, ver, plat, time.strptime(date, "%d-%b-%Y %H:%M")]
for path, ver, plat, date in salty_rx.findall(source)],
key=operator.itemgetter(3), reverse=True)
if version:
source_list = [s for s in source_list if s[1] == version]
if arch:
source_list = [s for s in source_list if s[2] == arch]
if not source_list:
return -1
version = source_list[0][1]
arch = source_list[0][2]
installer_url = base_url + source_list[0][0]
# It's no secret that Windows is notoriously command-line hostile.
# Win 7 and newer can use PowerShell out of the box, but to reach
# all those XP and 2K3 machines we must suppress our gag-reflex
# and use VB!
# The following script was borrowed from an informative article about
# downloading exploit payloads for malware. Nope, no irony here.
# http://www.greyhathacker.net/?p=500
vb_script = '''strFileURL = "{0}"
strHDLocation = "{1}"
Set objXMLHTTP = CreateObject("MSXML2.XMLHTTP")
objXMLHTTP.open "GET", strFileURL, false
objXMLHTTP.send()
If objXMLHTTP.Status = 200 Then
Set objADOStream = CreateObject("ADODB.Stream")
objADOStream.Open
objADOStream.Type = 1
objADOStream.Write objXMLHTTP.ResponseBody
objADOStream.Position = 0
objADOStream.SaveToFile strHDLocation
objADOStream.Close
Set objADOStream = Nothing
End if
Set objXMLHTTP = Nothing
Set objShell = CreateObject("WScript.Shell")
objShell.Exec("{1}{2}")'''
vb_saltexec = 'saltinstall.exe'
vb_saltexec_args = ' /S /minion-name=%COMPUTERNAME%'
if master:
vb_saltexec_args += ' /master={0}'.format(master)
# One further thing we need to do; the Windows Salt minion is pretty
# self-contained, except for the Microsoft Visual C++ 2008 runtime.
# It's tiny, so the bootstrap will attempt a silent install.
vb_vcrunexec = 'vcredist.exe'
if arch == 'AMD64':
vb_vcrun = vb_script.format(
'http://download.microsoft.com/download/d/2/4/d242c3fb-da5a-4542-ad66-f9661d0a8d19/vcredist_x64.exe',
vb_vcrunexec,
' /q')
else:
vb_vcrun = vb_script.format(
'http://download.microsoft.com/download/d/d/9/dd9a82d0-52ef-40db-8dab-795376989c03/vcredist_x86.exe',
vb_vcrunexec,
' /q')
vb_salt = vb_script.format(installer_url, vb_saltexec, vb_saltexec_args)
# PsExec doesn't like extra long arguments; save the instructions as a batch
# file so we can fire it over for execution.
# First off, change to the local temp directory, stop salt-minion (if
# running), and remove the master's public key.
# This is to accommodate for reinstalling Salt over an old or broken build,
# e.g. if the master address is changed, the salt-minion process will fail
# to authenticate and quit; which means infinite restarts under Windows.
batch = 'cd /d %TEMP%\nnet stop salt-minion\ndel c:\\salt\\conf\\pki\\minion\\minion_master.pub\n'
# Speaking of command-line hostile, cscript only supports reading a script
# from a file. Glue it together line by line.
for x, y in ((vb_vcrunexec, vb_vcrun), (vb_saltexec, vb_salt)):
vb_lines = y.split('\n')
batch += '\ndel ' + x + '\n@echo ' + vb_lines[0] + ' >' + \
x + '.vbs\n@echo ' + \
(' >>' + x + '.vbs\n@echo ').join(vb_lines[1:]) + \
' >>' + x + '.vbs\ncscript.exe /NoLogo ' + x + '.vbs'
batch_path = tempfile.mkstemp(suffix='.bat')[1]
with salt.utils.files.fopen(batch_path, 'wb') as batch_file:
batch_file.write(batch)
for host in hosts.split(","):
argv = ['psexec', '\\\\' + host]
if username:
argv += ['-u', username]
if password:
argv += ['-p', password]
argv += ['-h', '-c', batch_path]
subprocess.call(argv)
def upgrade_bootstrap(directory='.',
onlyif=None,
unless=None,
runas=None,
env=(),
offline=False,
buildout_ver=None):
'''
Upgrade current bootstrap.py with the last released one.
Indeed, when we first run a buildout, a common source of problem
is to have a locally stale bootstrap, we just try to grab a new copy
directory
directory to execute in
offline
are we executing buildout in offline mode
buildout_ver
forcing to use a specific buildout version (1 | 2)
onlyif
Only execute cmd if statement on the host return 0
unless
Do not execute cmd if statement on the host return 0
CLI Example:
.. code-block:: bash
salt '*' buildout.upgrade_bootstrap /srv/mybuildout
'''
if buildout_ver:
booturl = _URL_VERSIONS[buildout_ver]
else:
buildout_ver = _get_buildout_ver(directory)
booturl = _get_bootstrap_url(directory)
LOG.debug('Using {0}'.format(booturl))
# try to download an up-to-date bootstrap
# set defaulttimeout
# and add possible content
directory = os.path.abspath(directory)
b_py = os.path.join(directory, 'bootstrap.py')
comment = ''
try:
oldcontent = _get_bootstrap_content(directory)
dbuild = _dot_buildout(directory)
data = oldcontent
updated = False
dled = False
if not offline:
try:
if not os.path.isdir(dbuild):
os.makedirs(dbuild)
# only try to download once per buildout checkout
with salt.utils.fopen(os.path.join(
dbuild,
'{0}.updated_bootstrap'.format(buildout_ver))):
pass
except (OSError, IOError):
LOG.info('Bootstrap updated from repository')
data = _urlopen(booturl).read()
updated = True
dled = True
if 'socket.setdefaulttimeout' not in data:
updated = True
ldata = data.splitlines()
ldata.insert(1, 'import socket;socket.setdefaulttimeout(2)')
data = '\n'.join(ldata)
if updated:
comment = 'Bootstrap updated'
with salt.utils.fopen(b_py, 'w') as fic:
fic.write(data)
if dled:
with salt.utils.fopen(os.path.join(dbuild,
'{0}.updated_bootstrap'.format(
buildout_ver)), 'w') as afic:
afic.write('foo')
except (OSError, IOError):
if oldcontent:
with salt.utils.fopen(b_py, 'w') as fic:
fic.write(oldcontent)
return {'comment': comment}
