def test_request_from_http(self):
# We can't just call self.make_file because HTTPServerFixture will only
# serve content from the current WD. And we don't want to create random
# content in the original WD.
self.useFixture(TempWDFixture())
name = factory.getRandomString()
content = factory.getRandomString().encode('ascii')
factory.make_file(location='.', name=name, contents=content)
with HTTPServerFixture() as httpd:
url = urljoin(httpd.url, name)
response = MAASDispatcher().dispatch_query(url, {})
self.assertEqual(200, response.code)
self.assertEqual(content, response.read())
def test_request_from_http(self):
# We can't just call self.make_file because HTTPServerFixture will only
# serve content from the current WD. And we don't want to create random
# content in the original WD.
self.useFixture(TempWDFixture())
name = factory.getRandomString()
content = factory.getRandomString().encode('ascii')
factory.make_file(location='.', name=name, contents=content)
with HTTPServerFixture() as httpd:
url = urljoin(httpd.url, name)
response = MAASDispatcher().dispatch_query(url, {})
self.assertEqual(200, response.code)
self.assertEqual(content, response.read())
def test_no_autodetects_proxies(self):
self.open_func = lambda *args: MagicMock()
url = factory.make_url()
proxy_variables = {
"http_proxy": "http://proxy.example.com",
"https_proxy": "https://proxy.example.com",
"no_proxy": "noproxy.example.com",
}
with patch.dict(os.environ, proxy_variables):
dispatcher = MAASDispatcher(autodetect_proxies=False)
dispatcher.dispatch_query(url, {}, method="GET")
for handler in self.opener.handle_open["http"]:
if isinstance(handler, urllib.request.ProxyHandler):
raise AssertionError("ProxyHandler shouldn't be there")
def evaluate_tag(
system_id,
nodes,
tag_name,
tag_definition,
tag_nsmap,
credentials,
maas_url,
):
"""Evaluate `tag_definition` against this cluster's nodes' details.
:param system_id: System ID for the rack controller.
:param nodes: List of nodes to evaluate.
:param tag_name: The name of the tag, used for logging.
:param tag_definition: The XPath expression of the tag.
:param tag_nsmap: The namespace map as used by LXML's ETree library.
:param credentials: A 3-tuple of OAuth credentials.
:param maas_url: URL of the MAAS API.
"""
# Turn off proxy detection, since the rack should talk directly to
# the region, even if a system-wide proxy is configured.
client = MAASClient(
auth=MAASOAuth(*credentials),
dispatcher=MAASDispatcher(autodetect_proxies=False),
base_url=maas_url,
)
process_node_tags(
rack_id=system_id,
nodes=nodes,
tag_name=tag_name,
tag_definition=tag_definition,
tag_nsmap=tag_nsmap,
client=client,
)
def evaluate_tag(
system_id,
nodes,
tag_name,
tag_definition,
tag_nsmap,
credentials,
maas_url,
):
"""Evaluate `tag_definition` against this cluster's nodes' details.
:param system_id: System ID for the rack controller.
:param nodes: List of nodes to evaluate.
:param tag_name: The name of the tag, used for logging.
:param tag_definition: The XPath expression of the tag.
:param tag_nsmap: The namespace map as used by LXML's ETree library.
:param credentials: A 3-tuple of OAuth credentials.
:param maas_url: URL of the MAAS API.
"""
client = MAASClient(
auth=MAASOAuth(*credentials),
dispatcher=MAASDispatcher(),
base_url=maas_url,
)
process_node_tags(
rack_id=system_id,
nodes=nodes,
tag_name=tag_name,
tag_definition=tag_definition,
tag_nsmap=tag_nsmap,
client=client,
)
def submit(maas_url, api_credentials, images):
"""Submit images to server."""
MAASClient(MAASOAuth(*api_credentials), MAASDispatcher(),
maas_url).post('api/1.0/boot-images/',
'report_boot_images',
nodegroup=get_cluster_uuid(),
images=json.dumps(images))
def test_doesnt_override_accept_encoding_headers(self):
# If someone passes their own Accept-Encoding header, then dispatch
# just passes it through.
self.useFixture(TempWDFixture())
name = factory.getRandomString()
content = factory.getRandomString(300).encode('ascii')
factory.make_file(location='.', name=name, contents=content)
with HTTPServerFixture() as httpd:
url = urljoin(httpd.url, name)
headers = {'Accept-encoding': 'gzip'}
res = MAASDispatcher().dispatch_query(url, headers)
self.assertEqual(200, res.code)
self.assertEqual('gzip', res.info().get('Content-Encoding'))
raw_content = res.read()
read_content = gzip.GzipFile(
mode='rb', fileobj=BytesIO(raw_content)).read()
self.assertEqual(content, read_content)
def __init__(self, api_key=None, url=maas_url):
if api_key == None:
self.api_key = self.get_api_key()
else:
self.api_key = api_key
self.auth = MAASOAuth(*self.api_key.split(':'))
self.url = url
self.client = MAASClient(self.auth, MAASDispatcher(), self.url)
def test_doesnt_override_accept_encoding_headers(self):
# If someone passes their own Accept-Encoding header, then dispatch
# just passes it through.
self.useFixture(TempWDFixture())
name = factory.getRandomString()
content = factory.getRandomString(300).encode('ascii')
factory.make_file(location='.', name=name, contents=content)
with HTTPServerFixture() as httpd:
url = urljoin(httpd.url, name)
headers = {'Accept-encoding': 'gzip'}
res = MAASDispatcher().dispatch_query(url, headers)
self.assertEqual(200, res.code)
self.assertEqual('gzip', res.info().get('Content-Encoding'))
raw_content = res.read()
read_content = gzip.GzipFile(mode='rb',
fileobj=BytesIO(raw_content)).read()
self.assertEqual(content, read_content)
def test_dispatch_query_encodes_string_data(self):
# urllib, used by MAASDispatcher, requires data encoded into bytes. We
# encode into utf-8 in dispatch_query if necessary.
request = self.patch(urllib.request.Request, "__init__")
self.patch_urllib()
self.open_func = lambda *args: MagicMock()
url = factory.make_url()
data = factory.make_string(300, spaces=True)
MAASDispatcher().dispatch_query(url, {}, method="POST", data=data)
request.assert_called_once_with(ANY, url, bytes(data, "utf-8"), ANY)
def test_retries_three_times_on_503_service_unavailable(self):
self.useFixture(TempWDFixture())
name = factory.make_string()
content = factory.make_string().encode("ascii")
factory.make_file(location=".", name=name, contents=content)
with HTTPServerFixture() as httpd:
url = urljoin(httpd.url, name)
counter = {"count": 0}
def _wrap_open(*args, **kwargs):
if counter["count"] < 2:
counter["count"] += 1
raise urllib.error.HTTPError(url, 503,
"service unavailable", {},
None)
else:
return self.orig_open_func(*args, **kwargs)
self.open_func = _wrap_open
response = MAASDispatcher().dispatch_query(url, {})
self.assertEqual(200, response.code)
self.assertEqual(content, response.read())
def test_supports_content_encoding_gzip(self):
# The client will set the Accept-Encoding: gzip header, and it will
# also decompress the response if it comes back with Content-Encoding:
# gzip.
self.useFixture(TempWDFixture())
name = factory.make_string()
content = factory.make_string(300).encode('ascii')
factory.make_file(location='.', name=name, contents=content)
called = []
orig_urllib = urllib2.urlopen
def logging_urlopen(*args, **kwargs):
called.append((args, kwargs))
return orig_urllib(*args, **kwargs)
self.patch(urllib2, 'urlopen', logging_urlopen)
with HTTPServerFixture() as httpd:
url = urljoin(httpd.url, name)
res = MAASDispatcher().dispatch_query(url, {})
self.assertEqual(200, res.code)
self.assertEqual(content, res.read())
request = called[0][0][0]
self.assertEqual([((request,), {})], called)
self.assertEqual('gzip', request.headers.get('Accept-encoding'))
def update_region_controller(knowledge, interface, server):
"""Update the region controller with the status of the probe.
:param knowledge: dictionary of server info
:param interface: name of interface, e.g. eth0
:param server: IP address of detected DHCP server, or None
"""
api_path = 'api/1.0/nodegroups/%s/interfaces/%s/' % (
knowledge['nodegroup_uuid'], interface)
oauth = MAASOAuth(*knowledge['api_credentials'])
client = MAASClient(oauth, MAASDispatcher(), knowledge['maas_url'])
if server is None:
server = ''
process_request(client.put, api_path, foreign_dhcp_ip=server)
def test_supports_content_encoding_gzip(self):
# The client will set the Accept-Encoding: gzip header, and it will
# also decompress the response if it comes back with Content-Encoding:
# gzip.
self.useFixture(TempWDFixture())
name = factory.getRandomString()
content = factory.getRandomString(300).encode('ascii')
factory.make_file(location='.', name=name, contents=content)
called = []
orig_urllib = urllib2.urlopen
def logging_urlopen(*args, **kwargs):
called.append((args, kwargs))
return orig_urllib(*args, **kwargs)
self.patch(urllib2, 'urlopen', logging_urlopen)
with HTTPServerFixture() as httpd:
url = urljoin(httpd.url, name)
res = MAASDispatcher().dispatch_query(url, {})
self.assertEqual(200, res.code)
self.assertEqual(content, res.read())
request = called[0][0][0]
self.assertEqual([((request,), {})], called)
self.assertEqual('gzip', request.headers.get('Accept-encoding'))
def test_retries_three_times_on_503_service_unavailable(self):
self.useFixture(TempWDFixture())
name = factory.make_string()
content = factory.make_string().encode('ascii')
factory.make_file(location='.', name=name, contents=content)
with HTTPServerFixture() as httpd:
url = urljoin(httpd.url, name)
original_urlopen = urllib.request.urlopen
counter = {'count': 0}
def _wrap_urlopen(*args, **kwargs):
if counter['count'] < 2:
counter['count'] += 1
raise urllib.error.HTTPError(url, 503,
"service unavailable", {},
None)
else:
return original_urlopen(*args, **kwargs)
self.patch(urllib.request, "urlopen").side_effect = _wrap_urlopen
response = MAASDispatcher().dispatch_query(url, {})
self.assertEqual(200, response.code)
self.assertEqual(content, response.read())
def create_node(mac, arch, power_type, power_parameters):
api_credentials = get_recorded_api_credentials()
if api_credentials is None:
raise Exception('Not creating node: no API key yet.')
client = MAASClient(MAASOAuth(*api_credentials), MAASDispatcher(),
get_maas_url())
data = {
'op': 'new',
'architecture': arch,
'power_type': power_type,
'power_parameters': power_parameters,
'mac_addresses': mac
}
return client.post('/api/1.0/nodes/', data)
def get_cached_knowledge():
"""Get all the information that we need to know, or raise an error.
:return: (client, nodegroup_uuid)
"""
api_credentials = get_recorded_api_credentials()
if api_credentials is None:
logger.error("Not updating tags: don't have API key yet.")
return None, None
nodegroup_uuid = get_recorded_nodegroup_uuid()
if nodegroup_uuid is None:
logger.error("Not updating tags: don't have UUID yet.")
return None, None
client = MAASClient(MAASOAuth(*api_credentials), MAASDispatcher(),
get_maas_url())
return client, nodegroup_uuid
def test_retries_three_times_raises_503_service_unavailable(self):
self.useFixture(TempWDFixture())
name = factory.make_string()
content = factory.make_string().encode('ascii')
factory.make_file(location='.', name=name, contents=content)
with HTTPServerFixture() as httpd:
url = urljoin(httpd.url, name)
def _wrap_urlopen(*args, **kwargs):
raise urllib.error.HTTPError(url, 503, "service unavailable",
{}, None)
self.patch(urllib.request, "urlopen").side_effect = _wrap_urlopen
err = self.assertRaises(urllib.error.HTTPError,
MAASDispatcher().dispatch_query, url, {})
self.assertEqual(503, err.code)
def test_supports_any_method(self):
# urllib2, which MAASDispatcher uses, only supports POST and
# GET. There is some extra code that makes sure the passed
# method is honoured which is tested here.
self.useFixture(TempWDFixture())
name = factory.make_string()
content = factory.make_string(300).encode('ascii')
factory.make_file(location='.', name=name, contents=content)
method = "PUT"
# The test httpd doesn't like PUT, so we'll look for it bitching
# about that for the purposes of this test.
with HTTPServerFixture() as httpd:
url = urljoin(httpd.url, name)
e = self.assertRaises(
urllib2.HTTPError, MAASDispatcher().dispatch_query, url, {},
method=method)
self.assertIn("Unsupported method ('PUT')", e.reason)
def _getclient(url=u'http://localhost/MAAS/api/1.0/'):
'''
Use the MAAS apiclient to aquire a session with the Maas API
:param url:
How to connect to the Maas server. As we require the Maas tools
installed on the executing machine, this can be the localhost by default.
:return:
A MAASClient object
'''
global _mclient
if _mclient is None:
consumer_key, token, secret = key('root').split(':', 3)
auth = MAASOAuth(consumer_key, token, secret)
dispatch = MAASDispatcher()
_mclient = MAASClient(auth, dispatch, url)
return _mclient
def determine_cluster_interfaces(knowledge):
"""Given server knowledge, determine network interfaces on this cluster.
:return: a list of tuples of (interface name, ip) for all interfaces.
:note: this uses an API call and not local probing because the
region controller has the definitive and final say in what does and
doesn't exist.
"""
api_path = 'api/1.0/nodegroups/%s/interfaces' % knowledge['nodegroup_uuid']
oauth = MAASOAuth(*knowledge['api_credentials'])
client = MAASClient(oauth, MAASDispatcher(), knowledge['maas_url'])
interfaces = process_request(client.get, api_path, 'list')
if interfaces is None:
return None
interface_names = sorted((interface['interface'], interface['ip'])
for interface in interfaces
if interface['interface'] != '')
return interface_names
def test_autodetects_proxies(self):
self.open_func = lambda *args: MagicMock()
url = factory.make_url()
proxy_variables = {
"http_proxy": "http://proxy.example.com",
"https_proxy": "https://proxy.example.com",
"no_proxy": "noproxy.example.com",
}
with patch.dict(os.environ, proxy_variables):
MAASDispatcher().dispatch_query(url, {}, method="GET")
for handler in self.opener.handle_open["http"]:
if isinstance(handler, urllib.request.ProxyHandler):
break
else:
raise AssertionError("No ProxyHandler installed")
expected = {
"http": proxy_variables["http_proxy"],
"https": proxy_variables["https_proxy"],
"no": proxy_variables["no_proxy"],
}
for key, value in expected.items():
self.assertEqual(value, handler.proxies[key])
def send_leases(leases):
"""Send lease updates to the server API."""
# Items that the server must have sent us before we can do this.
knowledge = {
'maas_url': get_maas_url(),
'api_credentials': get_recorded_api_credentials(),
'nodegroup_uuid': get_recorded_nodegroup_uuid(),
}
if None in knowledge.values():
# The MAAS server hasn't sent us enough information for us to do
# this yet. Leave it for another time.
logger.info(
"Not sending DHCP leases to server: not all required knowledge "
"received from server yet. "
"Missing: %s" % ', '.join(list_missing_items(knowledge)))
return
api_path = 'api/1.0/nodegroups/%s/' % knowledge['nodegroup_uuid']
oauth = MAASOAuth(*knowledge['api_credentials'])
MAASClient(oauth, MAASDispatcher(),
knowledge['maas_url']).post(api_path,
'update_leases',
leases=json.dumps(leases))
def _auth(**connection_args):
'''
Set up maas credentials
Only intended to be used within maas-enabled modules
'''
prefix = "maas."
# look in connection_args first, then default to config file
def get(key, default=None):
return connection_args.get(
'connection_' + key, __salt__['config.get'](prefix + key, default))
api_token = get('token')
api_url = get('url', 'https://localhost/')
LOG.debug("MAAS url: " + api_url)
LOG.debug("MAAS token: " + api_token)
auth = MAASOAuth(*api_token.split(":"))
dispatcher = MAASDispatcher()
client = MAASClient(auth, dispatcher, api_url)
return client
def test_dispatch_query_makes_direct_call(self):
contents = factory.getRandomString()
url = "file://%s" % self.make_file(contents=contents)
self.assertEqual(contents,
MAASDispatcher().dispatch_query(url, {}).read())
def _connect(self):
if not self.conn:
auth = MAASOAuth(*self.maas_api_key.split(':'))
dispatcher = MAASDispatcher()
self.maas_client = MAASClient(auth, dispatcher, self.maas_url)
self.conn = True
def get_client(url, creds):
[consumer_key, token, secret] = creds.split(':')
auth = MAASOAuth(consumer_key=consumer_key,
resource_token=token,
resource_secret=secret)
return MAASClient(auth, MAASDispatcher(), url)
def _getclient(url=u'http://localhost/MAAS/api/1.0/'):
consumer_key, token, secret = key('root').split(':', 3)
auth = MAASOAuth(consumer_key, token, secret)
dispatch = MAASDispatcher()
client = MAASClient(auth, dispatch, url)
return client
def make_anonymous_api_client(server_url):
"""Create an unauthenticated API client."""
return MAASClient(NoAuth(), MAASDispatcher(), server_url)
