def _remove_add_servers_from_queryenv(self):
cnf = ScalarizrCnf(bus.etc_path)
cnf.bootstrap()
globals()['ini'] = cnf.rawini
key_path = os.path.join(bus.etc_path, ini.get('general', 'crypto_key_path'))
server_id = ini.get('general', 'server_id')
url = ini.get('general','queryenv_url')
queryenv = QueryEnvService(url, server_id, key_path)
result = queryenv.list_roles()
running_servers = []
bnds = []
for elem in self.api.list_listeners():
bnds.append(elem['backend'])
bnds = list(set(bnds))
for bnd in bnds:
for srv in self.api.list_servers(backend=bnd):
self.api.remove_server(ipaddr=srv, backend=bnd)
for d in result:
behaviour=', '.join(d.behaviour)
for host in d.hosts:
try:
if 'role:%s' % d.farm_role_id in bnds:
self.api.add_server(ipaddr=host.internal_ip,
backend='role:%s' % d.farm_role_id)
except:
LOG.warn('HAProxyHandler.on_before_host_up.Failed add_server `%s` in'
' backend=`role:%s`, details: %s' % (
host.internal_ip.replace('.', '-'),
d.farm_role_id, sys.exc_info()[1]))
running_servers.append([d.farm_role_id, host.internal_ip])
LOG.debug('running_servers: `%s`', running_servers)
def new_queryenv():
queryenv_creds = (__node__['queryenv_url'],
__node__['server_id'],
os.path.join(scalr_base_dir, __node__['crypto_key_path']))
queryenv = QueryEnvService(*queryenv_creds)
api_version = queryenv.get_latest_version()
queryenv = QueryEnvService(*queryenv_creds, api_version=api_version, autoretry=False)
return queryenv
def new_queryenv():
init_cnf()
key_path = os.path.join(bus.etc_path, ini.get('general',
'crypto_key_path'))
server_id = ini.get('general', 'server_id')
url = ini.get('general', 'queryenv_url')
api_version = QUERYENV_API_VERSION
if not api_version:
cnf = bus.cnf
if not bus.scalr_version:
version_file = cnf.private_path('.scalr-version')
if os.path.exists(version_file):
with open(version_file, 'r') as fp:
bus.scalr_version = tuple(fp.read().strip().split('.'))
if bus.scalr_version:
if bus.scalr_version >= (3, 5, 3):
api_version = '2012-07-01'
elif bus.scalr_version >= (3, 1, 0):
api_version = '2012-04-17'
else:
api_version = '2010-09-23'
else:
api_version = '2012-07-01'
return QueryEnvService(url, server_id, key_path, api_version)
def _init_services():
logger = logging.getLogger(__name__)
cnf = bus.cnf; ini = cnf.rawini
server_id = ini.get('general', 'server_id')
queryenv_url = ini.get('general', 'queryenv_url')
messaging_adp = ini.get('messaging', 'adapter')
# Set base URL
pr = urlparse(queryenv_url)
bus.scalr_url = urlunparse((pr.scheme, pr.netloc, '', '', '', ''))
logger.debug("Got scalr url: '%s'" % bus.scalr_url)
# Create periodical executor for background tasks (cleanup, rotate, gc, etc...)
bus.periodical_executor = PeriodicalExecutor()
logger.debug("Initialize QueryEnv client")
queryenv = QueryEnvService(queryenv_url, server_id, cnf.key_path(cnf.DEFAULT_KEY), '2008-12-16')
queryenv = QueryEnvService(queryenv_url, server_id, cnf.key_path(cnf.DEFAULT_KEY), queryenv.get_latest_version())
bus.queryenv_service = queryenv
bus.queryenv_version = tuple(map(int, queryenv.api_version.split('-')))
logger.debug("Initialize messaging")
factory = MessageServiceFactory()
try:
params = dict(ini.items("messaging_" + messaging_adp))
params[P2pConfigOptions.SERVER_ID] = server_id
params[P2pConfigOptions.CRYPTO_KEY_PATH] = cnf.key_path(cnf.DEFAULT_KEY)
msg_service = factory.new_service(messaging_adp, **params)
bus.messaging_service = msg_service
except (BaseException, Exception):
raise ScalarizrError("Cannot create messaging service adapter '%s'" % (messaging_adp))
logger.debug('Initialize message handlers')
consumer = msg_service.get_consumer()
consumer.listeners.append(MessageListener())
logger.debug('Schedule SNMP process')
globals()['_snmp_scheduled_start_time'] = time.time()
Storage.maintain_volume_table = True
if not bus.api_server:
api_app = jsonrpc_http.WsgiApplication(rpc.RequestHandler(_api_routes),
cnf.key_path(cnf.DEFAULT_KEY))
bus.api_server = wsgiref.simple_server.make_server('0.0.0.0', 8010, api_app)
def _remove_add_servers_from_queryenv(self):
cnf = ScalarizrCnf(bus.etc_path)
cnf.bootstrap()
globals()['ini'] = cnf.rawini
key_path = os.path.join(bus.etc_path,
ini.get('general', 'crypto_key_path'))
server_id = ini.get('general', 'server_id')
url = ini.get('general', 'queryenv_url')
queryenv = QueryEnvService(url, server_id, key_path)
result = queryenv.list_roles()
running_servers = []
bnds = []
for elem in self.api.list_listeners():
bnds.append(elem['backend'])
bnds = list(set(bnds))
for bnd in bnds:
for srv in self.api.list_servers(backend=bnd):
self.api.remove_server(ipaddr=srv, backend=bnd)
for d in result:
behaviour = ', '.join(d.behaviour)
for host in d.hosts:
try:
if 'role:%s' % d.farm_role_id in bnds:
self.api.add_server(ipaddr=host.internal_ip,
backend='role:%s' % d.farm_role_id)
except:
LOG.warn(
'HAProxyHandler.on_before_host_up.Failed add_server `%s` in'
' backend=`role:%s`, details: %s' %
(host.internal_ip.replace(
'.', '-'), d.farm_role_id, sys.exc_info()[1]))
running_servers.append([d.farm_role_id, host.internal_ip])
LOG.debug('running_servers: `%s`', running_servers)
def prepare_snmp():
_init()
cnf = bus.cnf
ini = cnf.rawini
cnf.on('apply_user_data', _apply_user_data)
cnf.bootstrap()
server_id = ini.get('general', 'server_id')
queryenv_url = ini.get('general', 'queryenv_url')
queryenv = QueryEnvService(queryenv_url, server_id,
cnf.key_path(cnf.DEFAULT_KEY))
bus.queryenv_service = queryenv
snmp_server = SnmpServer(
port=int(ini.get(config.SECT_SNMP, config.OPT_PORT)),
security_name=ini.get(config.SECT_SNMP, config.OPT_SECURITY_NAME),
community_name=ini.get(config.SECT_SNMP, config.OPT_COMMUNITY_NAME))
return snmp_server
def _init_services():
logger = logging.getLogger(__name__)
cnf = bus.cnf
ini = cnf.rawini
server_id = ini.get('general', 'server_id')
queryenv_url = ini.get('general', 'queryenv_url')
messaging_adp = ini.get('messaging', 'adapter')
# Set base URL
pr = urlparse(queryenv_url)
bus.scalr_url = urlunparse((pr.scheme, pr.netloc, '', '', '', ''))
logger.debug("Got scalr url: '%s'" % bus.scalr_url)
# Create periodical executor for background tasks (cleanup, rotate, gc, etc...)
bus.periodical_executor = PeriodicalExecutor()
logger.debug("Initialize QueryEnv client")
queryenv = QueryEnvService(queryenv_url, server_id,
cnf.key_path(cnf.DEFAULT_KEY), '2008-12-16')
queryenv_latest = queryenv.get_latest_version()
queryenv = QueryEnvService(queryenv_url, server_id,
cnf.key_path(cnf.DEFAULT_KEY), queryenv_latest)
if tuple(map(int, queryenv_latest.split('-'))) >= (2012, 7, 1):
scalr_version = queryenv.get_global_config()['params'].get(
'scalr.version')
if scalr_version:
bus.scalr_version = tuple(map(int, scalr_version.split('.')))
version_file = cnf.private_path('.scalr-version')
with open(version_file, 'w') as fp:
fp.write(scalr_version)
bus.queryenv_service = queryenv
bus.queryenv_version = tuple(map(int, queryenv.api_version.split('-')))
logger.debug("Initialize messaging")
factory = MessageServiceFactory()
try:
params = dict(ini.items("messaging_" + messaging_adp))
params[P2pConfigOptions.SERVER_ID] = server_id
params[P2pConfigOptions.CRYPTO_KEY_PATH] = cnf.key_path(
cnf.DEFAULT_KEY)
msg_service = factory.new_service(messaging_adp, **params)
bus.messaging_service = msg_service
except (BaseException, Exception):
raise ScalarizrError("Cannot create messaging service adapter '%s'" %
(messaging_adp))
logger.debug('Initialize message handlers')
consumer = msg_service.get_consumer()
consumer.listeners.append(MessageListener())
logger.debug('Schedule SNMP process')
globals()['_snmp_scheduled_start_time'] = time.time()
Storage.maintain_volume_table = True
if not bus.api_server:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
api_port = 8010
try:
sock.connect(('0.0.0.0', api_port))
STATE['global.api_port'] = api_port = 8009
sock.close()
except socket.error:
pass
api_app = jsonrpc_http.WsgiApplication(rpc.RequestHandler(_api_routes),
cnf.key_path(cnf.DEFAULT_KEY))
bus.api_server = wsgiref.simple_server.make_server(
'0.0.0.0', api_port, api_app)
def _init_services(self):
logger = logging.getLogger(__name__)
cnf = bus.cnf
ini = cnf.rawini
server_id = ini.get('general', 'server_id')
queryenv_url = ini.get('general', 'queryenv_url')
messaging_adp = ini.get('messaging', 'adapter')
# Set base URL
pr = urlparse(queryenv_url)
bus.scalr_url = urlunparse((pr.scheme, pr.netloc, '', '', '', ''))
logger.debug("Got scalr url: '%s'" % bus.scalr_url)
if not linux.os.windows and node.__node__['platform'].name in (
'eucalyptus', 'openstack'):
self._try_resolver(bus.scalr_url)
# Create periodical executor for background tasks (cleanup, rotate, gc, etc...)
bus.periodical_executor = PeriodicalExecutor()
logger.debug("Initialize QueryEnv client")
queryenv = QueryEnvService(queryenv_url, server_id,
cnf.key_path(cnf.DEFAULT_KEY), '2008-12-16')
queryenv_latest = queryenv.get_latest_version()
queryenv = QueryEnvService(queryenv_url, server_id,
cnf.key_path(cnf.DEFAULT_KEY),
queryenv_latest)
if tuple(map(int, queryenv_latest.split('-'))) >= (2012, 7, 1):
scalr_version = queryenv.get_global_config()['params'].get(
'scalr.version')
if scalr_version:
bus.scalr_version = tuple(map(int, scalr_version.split('.')))
version_file = cnf.private_path('.scalr-version')
with open(version_file, 'w') as fp:
fp.write(scalr_version)
bus.queryenv_service = queryenv
bus.queryenv_version = tuple(map(int, queryenv.api_version.split('-')))
ports_non_default = self._select_control_ports()
logger.debug("Initialize messaging")
factory = MessageServiceFactory()
try:
params = dict(ini.items("messaging_" + messaging_adp))
if ports_non_default:
consumer_url = list(
urlparse(params[P2pConfigOptions.CONSUMER_URL]))
consumer_url[1] = ':'.join(
(consumer_url[1].split(':')[0],
str(node.__node__['base']['messaging_port'])))
params[P2pConfigOptions.CONSUMER_URL] = urlunparse(
consumer_url)
params[P2pConfigOptions.SERVER_ID] = server_id
params[P2pConfigOptions.CRYPTO_KEY_PATH] = cnf.key_path(
cnf.DEFAULT_KEY)
msg_service = factory.new_service(messaging_adp, **params)
bus.messaging_service = msg_service
except (BaseException, Exception):
raise ScalarizrError(
"Cannot create messaging service adapter '%s'" %
(messaging_adp))
logger.debug('Initialize message handlers')
consumer = msg_service.get_consumer()
consumer.listeners.append(MessageListener())
if not linux.os.windows_family:
logger.debug('Schedule SNMP process')
self._snmp_scheduled_start_time = time.time()
Storage.maintain_volume_table = True
if not bus.api_server:
api_app = jsonrpc_http.WsgiApplication(
rpc.RequestHandler(_api_routes), cnf.key_path(cnf.DEFAULT_KEY))
class ThreadingWSGIServer(SocketServer.ThreadingMixIn,
wsgiref.simple_server.WSGIServer):
pass
bus.api_server = wsgiref.simple_server.make_server(
'0.0.0.0',
node.__node__['base']['api_port'],
api_app,
server_class=ThreadingWSGIServer)
if ports_non_default:
msg = msg_service.new_message(
'HostUpdate', None, {
'base': {
'api_port': node.__node__['base']['api_port'],
'messaging_port':
node.__node__['base']['messaging_port']
}
})
msg_service.get_producer().send(Queues.CONTROL, msg)
def setUp (self):
self._queryenv = QueryEnvService("http://ec2farm-dev.bsd2.webta.local/query-env/","127","c+/g0PyouaqXMbuJ5Vtux34Mft7jLe5H5u8tUmyhldjwTfgm7BI6MOA8F6BwkzQnpWEOcHx+A+TRJh0u3PElQQ0SiwdwrlgpQMbj8NBxbxBgfxA9WisgvfQu5ZPYou6Gz3oUAQdWfFlFdY2ACOjmqa3DGogge+TlXtV2Xagm0rw=")
class Test(unittest.TestCase):
def setUp (self):
self._queryenv = QueryEnvService("http://ec2farm-dev.bsd2.webta.local/query-env/","127","c+/g0PyouaqXMbuJ5Vtux34Mft7jLe5H5u8tUmyhldjwTfgm7BI6MOA8F6BwkzQnpWEOcHx+A+TRJh0u3PElQQ0SiwdwrlgpQMbj8NBxbxBgfxA9WisgvfQu5ZPYou6Gz3oUAQdWfFlFdY2ACOjmqa3DGogge+TlXtV2Xagm0rw=")
def test_get_service_configuration_response(self):
xmlfile = get_xml_file("get-service-configuration_response.xml")
xml = xml_strip(parseString(open(xmlfile, "r").read()))
ret = self._queryenv._read_get_service_configuration_response(xml, 'app')
self.assertFalse(ret is None)
self.assertTrue(type(ret.settings) is dict)
self.assertTrue(len(ret.settings))
self.assertTrue('__defaults__' in ret.settings)
self.assertTrue(type(ret.name) is type(""))
self.assertEqual(ret.name, "app-test")
def test_get_latest_version_response(self):
xmlfile = get_xml_file("get_latest_version_response.xml")
xml = xml_strip(parseString(open(xmlfile, "r").read()))
version = self._queryenv._read_get_latest_version_response(xml)
self.assertFalse(version is None)
self.assertEqual(version, "2009-03-05")
def test_get_https_certificate_response(self):
xmlfile = get_xml_file("get_https_certificate_response.xml")
xml = parseString(open(xmlfile, "r").read())
xml = xml_strip(xml)
cert = self._queryenv._read_get_https_certificate_response(xml)
self.assertFalse(cert is (None,None))
self.assertEqual(cert[0], """-----BEGIN CERTIFICATE-----
MIICATCCAWoCCQDVWoPxl2kzdzANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJB
VTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0
cyBQdHkgTHRkMB4XDTEwMDYwNDEyMDcyNloXDTExMDYwNDEyMDcyNlowRTELMAkG
A1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNVBAoTGEludGVybmV0
IFdpZGdpdHMgUHR5IEx0ZDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA5DeJ
DnVd/Jgb3bPh296l78V/T7kRG8IAejawlR25ikUXpG9DoaV05nDwDR+e3Hm/wvmY
7EwqOiVFtFc1Uoc6av34zIroEcGmwi+nnUAptpHOC/863VhuSehKf7lWkUbLz8OS
NAJOj5c6jxKfasnkqxYEakhvXCe/N4NQWdnSaVUCAwEAATANBgkqhkiG9w0BAQUF
AAOBgQAJ318EcccEM34pFsF5thGQfYU4yrTb2P+Xyg/bhNgByCc+agWh9MAXBF1G
EEH5rvsHc1ocVXi69a45D+m0pV5ZJwSIrwo6ssViMpWmUfIStkmYm7qsRbNnIWkZ
TuMUjy2djQJdeAadKNinJ5YXk2iU7XvxTzaqZAzdpv2/9G5Nbg==
-----END CERTIFICATE-----
""")
self.assertEqual(cert[1], """-----BEGIN RSA PRIVATE KEY-----
MIICXgIBAAKBgQDkN4kOdV38mBvds+Hb3qXvxX9PuREbwgB6NrCVHbmKRRekb0Oh
pXTmcPANH57ceb/C+ZjsTCo6JUW0VzVShzpq/fjMiugRwabCL6edQCm2kc4L/zrd
WG5J6Ep/uVaRRsvPw5I0Ak6PlzqPEp9qyeSrFgRqSG9cJ783g1BZ2dJpVQIDAQAB
AoGBAKNtDY24CIUBHDtoPF4qE6QfRur9OT4qcBMUpOEztxFIJwUO1FymUo9O4uhS
830pBmSGPrdAV6Dp3f+lz754riBj1Hzk5kQuF6rAxoBspbqXQB30Pz1r6qWt9Sf2
DZqQ9278UZEtUQq90QzEjh3xAV5BxG1Qv+d3yyIVy1K5Pt7BAkEA957XKSdGpQW5
c2V6rUnfvLi/WeIx8xFFL1ohnPSBLhvP8Hc4zJGVVBnuX9RKEVVLCizRBLFK93uI
nh0aGSa9hQJBAOvwmamWMkFH4aYymqZY5CRooxCG6Wv8MTQBhgGaLyGYX/CaHpRf
Y5RCEWnXZJr/rAwnowLr8kh/MiZGIBWVHZECQQDBeSRYDU4PRke+OD4AA8aC6D7q
defdKVNLSjsVLZ15b1WrZxvECsQIcDJmQbKVlHULQDUYW4Zdk/IMyGRJ3pEZAkEA
jWaz4RQX6FHZJY7cameJy1w+phAE4ufQ4TcshddO+dZlYUAspYWJm3gBEaq6K76g
8OPsaTrZCKPafV+3qNemUQJACZ7FDJKmO9SLccpYIDTcMIhqgu2QseZwjJPjAMbg
0xHR3hyeMnkrjP9amXrbAxOwndCD10I6Tuw4Qj7t20p+hw==
-----END RSA PRIVATE KEY-----
""")
def test_list_roles(self):
xmlfile = get_xml_file("list_roles_response.xml")
xml = parseString(open(xmlfile, "r").read())
#print open(xmlfile, "r").read()
xml = xml_strip(xml)
roles = self._queryenv._read_list_roles_response(xml)
role = roles[0]
self.assertFalse(roles is None)
self.assertEqual(len(roles), 2)
self.assertEqual(role.behaviour, ['app', "mysql"])
self.assertEqual(role.name, "lamp-custom")
hosts = role.hosts
host = hosts[0]
self.assertFalse(hosts is None)
self.assertEqual(host.internal_ip, "211.31.14.198")
self.assertEqual(host.external_ip, "211.31.14.198")
self.assertTrue(host.replication_master)
self.assertEqual(host.index, 1)
def test_read_list_ebs_mountpoints_response(self):
xmlfile = get_xml_file("list_ebs_mountpoints_response.xml")
xml = parseString(open(xmlfile, "r").read())
xml = xml_strip(xml)
mountpoints = self._queryenv._read_list_ebs_mountpoints_response(xml)
mountpoint = mountpoints[0]
volumes = mountpoint.volumes
volume = volumes[0]
self.assertFalse(mountpoints is None)
self.assertEqual(len(mountpoints), 2)
self.assertEqual(mountpoint.name, "some_name_for_LVM")
self.assertEqual(mountpoint.dir, "/mnt/storage1")
self.assertTrue(mountpoint.create_fs)
self.assertFalse(mountpoint.is_array)
self.assertFalse(volumes is None)
self.assertEqual(volume.volume_id, "vol-123451")
self.assertEqual(volume.device, "/dev/sdb")
def test_list_role_params(self):
xmlfile = get_xml_file("list_role_params_response.xml")
xml = parseString(open(xmlfile, "r").read())
xml = xml_strip(xml)
parameters = self._queryenv._read_list_role_params_response(xml)
self.assertFalse(parameters is None)
#self.assertTrue(parametres.has_key("external_ips_to_allow_access_from"))
self.assertEqual(parameters["external_ips_to_allow_access_from"].strip(), '')
def test_read_list_scripts_response(self):
xmlfile = get_xml_file("list_scripts_response.xml")
xml = parseString(open(xmlfile, "r").read())
xml = xml_strip(xml)
scripts = self._queryenv._read_list_scripts_response(xml)
script = scripts[0]
self.assertFalse(scripts is None)
self.assertEqual(len(scripts), 1)
self.assertTrue(script.asynchronous)
self.assertEqual(script.exec_timeout, 100)
self.assertEqual(script.name, 'script_name')
self.assertEqual(script.body.strip(), '')
def test_read_list_virtualhosts_response(self):
xmlfile = get_xml_file("list_virtualhosts_response.xml")
xml = parseString(open(xmlfile, "r").read())
xml = xml_strip(xml)
vhosts = self._queryenv._read_list_virtualhosts_response(xml)
vhost = vhosts[0]
self.assertFalse(vhosts is None)
self.assertEqual(len(vhosts), 2)
self.assertEqual(vhost.hostname, 'gpanel.net')
self.assertEqual(vhost.type, 'apache')
self.assertEqual(vhost.raw, '''
''')
self.assertTrue(vhosts[1].https)
def test__read_list_farm_role_params_response(self):
xmlfile = get_xml_file("list_farm_role_params_response.xml")
xml = parseString(open(xmlfile, "r").read())
xml = xml_strip(xml)
params = self._queryenv._read_list_farm_role_params_response(xml)
print params
def test_xml2dict(self):
result = {'mysql': {'root_password': '******', 'stat_password': '******', 'volume_config': {'device': '/dev/xvdp', 'mpoint': '/mnt/dbstorage', 'type': 'ebs', 'id': 'vol-12345678', 'size': '100'}, 'log_file': 'binlog.000003', 'repl_password': '******', 'log_pos': '106'}}
xmlfile = get_xml_file('xml2dict.xml')
self.assertEqual(xml_file_to_dict(xmlfile), result)
print 'done'
def setUp(self):
self._queryenv = QueryEnvService(
"http://ec2farm-dev.bsd2.webta.local/query-env/", "127",
"c+/g0PyouaqXMbuJ5Vtux34Mft7jLe5H5u8tUmyhldjwTfgm7BI6MOA8F6BwkzQnpWEOcHx+A+TRJh0u3PElQQ0SiwdwrlgpQMbj8NBxbxBgfxA9WisgvfQu5ZPYou6Gz3oUAQdWfFlFdY2ACOjmqa3DGogge+TlXtV2Xagm0rw="
)
class Test(unittest.TestCase):
def setUp(self):
self._queryenv = QueryEnvService(
"http://ec2farm-dev.bsd2.webta.local/query-env/", "127",
"c+/g0PyouaqXMbuJ5Vtux34Mft7jLe5H5u8tUmyhldjwTfgm7BI6MOA8F6BwkzQnpWEOcHx+A+TRJh0u3PElQQ0SiwdwrlgpQMbj8NBxbxBgfxA9WisgvfQu5ZPYou6Gz3oUAQdWfFlFdY2ACOjmqa3DGogge+TlXtV2Xagm0rw="
)
def test_get_service_configuration_response(self):
xmlfile = get_xml_file("get-service-configuration_response.xml")
xml = xml_strip(parseString(open(xmlfile, "r").read()))
ret = self._queryenv._read_get_service_configuration_response(
xml, 'app')
self.assertFalse(ret is None)
self.assertTrue(type(ret.settings) is dict)
self.assertTrue(len(ret.settings))
self.assertTrue('__defaults__' in ret.settings)
self.assertTrue(type(ret.name) is type(""))
self.assertEqual(ret.name, "app-test")
def test_get_latest_version_response(self):
xmlfile = get_xml_file("get_latest_version_response.xml")
xml = xml_strip(parseString(open(xmlfile, "r").read()))
version = self._queryenv._read_get_latest_version_response(xml)
self.assertFalse(version is None)
self.assertEqual(version, "2009-03-05")
def test_get_https_certificate_response(self):
xmlfile = get_xml_file("get_https_certificate_response.xml")
xml = parseString(open(xmlfile, "r").read())
xml = xml_strip(xml)
cert = self._queryenv._read_get_https_certificate_response(xml)
self.assertFalse(cert is (None, None))
self.assertEqual(
cert[0], """-----BEGIN CERTIFICATE-----
MIICATCCAWoCCQDVWoPxl2kzdzANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJB
VTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0
cyBQdHkgTHRkMB4XDTEwMDYwNDEyMDcyNloXDTExMDYwNDEyMDcyNlowRTELMAkG
A1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNVBAoTGEludGVybmV0
IFdpZGdpdHMgUHR5IEx0ZDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA5DeJ
DnVd/Jgb3bPh296l78V/T7kRG8IAejawlR25ikUXpG9DoaV05nDwDR+e3Hm/wvmY
7EwqOiVFtFc1Uoc6av34zIroEcGmwi+nnUAptpHOC/863VhuSehKf7lWkUbLz8OS
NAJOj5c6jxKfasnkqxYEakhvXCe/N4NQWdnSaVUCAwEAATANBgkqhkiG9w0BAQUF
AAOBgQAJ318EcccEM34pFsF5thGQfYU4yrTb2P+Xyg/bhNgByCc+agWh9MAXBF1G
EEH5rvsHc1ocVXi69a45D+m0pV5ZJwSIrwo6ssViMpWmUfIStkmYm7qsRbNnIWkZ
TuMUjy2djQJdeAadKNinJ5YXk2iU7XvxTzaqZAzdpv2/9G5Nbg==
-----END CERTIFICATE-----
""")
self.assertEqual(
cert[1], """-----BEGIN RSA PRIVATE KEY-----
MIICXgIBAAKBgQDkN4kOdV38mBvds+Hb3qXvxX9PuREbwgB6NrCVHbmKRRekb0Oh
pXTmcPANH57ceb/C+ZjsTCo6JUW0VzVShzpq/fjMiugRwabCL6edQCm2kc4L/zrd
WG5J6Ep/uVaRRsvPw5I0Ak6PlzqPEp9qyeSrFgRqSG9cJ783g1BZ2dJpVQIDAQAB
AoGBAKNtDY24CIUBHDtoPF4qE6QfRur9OT4qcBMUpOEztxFIJwUO1FymUo9O4uhS
830pBmSGPrdAV6Dp3f+lz754riBj1Hzk5kQuF6rAxoBspbqXQB30Pz1r6qWt9Sf2
DZqQ9278UZEtUQq90QzEjh3xAV5BxG1Qv+d3yyIVy1K5Pt7BAkEA957XKSdGpQW5
c2V6rUnfvLi/WeIx8xFFL1ohnPSBLhvP8Hc4zJGVVBnuX9RKEVVLCizRBLFK93uI
nh0aGSa9hQJBAOvwmamWMkFH4aYymqZY5CRooxCG6Wv8MTQBhgGaLyGYX/CaHpRf
Y5RCEWnXZJr/rAwnowLr8kh/MiZGIBWVHZECQQDBeSRYDU4PRke+OD4AA8aC6D7q
defdKVNLSjsVLZ15b1WrZxvECsQIcDJmQbKVlHULQDUYW4Zdk/IMyGRJ3pEZAkEA
jWaz4RQX6FHZJY7cameJy1w+phAE4ufQ4TcshddO+dZlYUAspYWJm3gBEaq6K76g
8OPsaTrZCKPafV+3qNemUQJACZ7FDJKmO9SLccpYIDTcMIhqgu2QseZwjJPjAMbg
0xHR3hyeMnkrjP9amXrbAxOwndCD10I6Tuw4Qj7t20p+hw==
-----END RSA PRIVATE KEY-----
""")
def test_list_roles(self):
xmlfile = get_xml_file("list_roles_response.xml")
xml = parseString(open(xmlfile, "r").read())
#print open(xmlfile, "r").read()
xml = xml_strip(xml)
roles = self._queryenv._read_list_roles_response(xml)
role = roles[0]
self.assertFalse(roles is None)
self.assertEqual(len(roles), 2)
self.assertEqual(role.behaviour, ['app', "mysql"])
self.assertEqual(role.name, "lamp-custom")
hosts = role.hosts
host = hosts[0]
self.assertFalse(hosts is None)
self.assertEqual(host.internal_ip, "211.31.14.198")
self.assertEqual(host.external_ip, "211.31.14.198")
self.assertTrue(host.replication_master)
self.assertEqual(host.index, 1)
def test_read_list_ebs_mountpoints_response(self):
xmlfile = get_xml_file("list_ebs_mountpoints_response.xml")
xml = parseString(open(xmlfile, "r").read())
xml = xml_strip(xml)
mountpoints = self._queryenv._read_list_ebs_mountpoints_response(xml)
mountpoint = mountpoints[0]
volumes = mountpoint.volumes
volume = volumes[0]
self.assertFalse(mountpoints is None)
self.assertEqual(len(mountpoints), 2)
self.assertEqual(mountpoint.name, "some_name_for_LVM")
self.assertEqual(mountpoint.dir, "/mnt/storage1")
self.assertTrue(mountpoint.create_fs)
self.assertFalse(mountpoint.is_array)
self.assertFalse(volumes is None)
self.assertEqual(volume.volume_id, "vol-123451")
self.assertEqual(volume.device, "/dev/sdb")
def test_list_role_params(self):
xmlfile = get_xml_file("list_role_params_response.xml")
xml = parseString(open(xmlfile, "r").read())
xml = xml_strip(xml)
parameters = self._queryenv._read_list_role_params_response(xml)
self.assertFalse(parameters is None)
#self.assertTrue(parametres.has_key("external_ips_to_allow_access_from"))
self.assertEqual(
parameters["external_ips_to_allow_access_from"].strip(), '')
def test_read_list_scripts_response(self):
xmlfile = get_xml_file("list_scripts_response.xml")
xml = parseString(open(xmlfile, "r").read())
xml = xml_strip(xml)
scripts = self._queryenv._read_list_scripts_response(xml)
script = scripts[0]
self.assertFalse(scripts is None)
self.assertEqual(len(scripts), 1)
self.assertTrue(script.asynchronous)
self.assertEqual(script.exec_timeout, 100)
self.assertEqual(script.name, 'script_name')
self.assertEqual(script.body.strip(), '')
def test_read_list_virtualhosts_response(self):
xmlfile = get_xml_file("list_virtualhosts_response.xml")
xml = parseString(open(xmlfile, "r").read())
xml = xml_strip(xml)
vhosts = self._queryenv._read_list_virtualhosts_response(xml)
vhost = vhosts[0]
self.assertFalse(vhosts is None)
self.assertEqual(len(vhosts), 2)
self.assertEqual(vhost.hostname, 'gpanel.net')
self.assertEqual(vhost.type, 'apache')
self.assertEqual(vhost.raw, '''
''')
self.assertTrue(vhosts[1].https)
def test__read_list_farm_role_params_response(self):
xmlfile = get_xml_file("list_farm_role_params_response.xml")
xml = parseString(open(xmlfile, "r").read())
xml = xml_strip(xml)
params = self._queryenv._read_list_farm_role_params_response(xml)
print params
def test_xml2dict(self):
result = {
'mysql': {
'root_password': '******',
'stat_password': '******',
'volume_config': {
'device': '/dev/xvdp',
'mpoint': '/mnt/dbstorage',
'type': 'ebs',
'id': 'vol-12345678',
'size': '100'
},
'log_file': 'binlog.000003',
'repl_password': '******',
'log_pos': '106'
}
}
xmlfile = get_xml_file('xml2dict.xml')
self.assertEqual(xml_file_to_dict(xmlfile), result)
print 'done'
def _init_services(self):
logger = logging.getLogger(__name__)
cnf = bus.cnf
ini = cnf.rawini
server_id = ini.get('general', 'server_id')
queryenv_url = ini.get('general', 'queryenv_url')
messaging_adp = ini.get('messaging', 'adapter')
# Set base URL
pr = urlparse(queryenv_url)
bus.scalr_url = urlunparse((pr.scheme, pr.netloc, '', '', '', ''))
logger.debug("Got scalr url: '%s'" % bus.scalr_url)
if not linux.os.windows and __node__['platform'].name == 'openstack':
self._try_resolver(bus.scalr_url)
# Create periodical executor for background tasks (cleanup, rotate, gc, etc...)
bus.periodical_executor = PeriodicalExecutor()
logger.debug("Initialize QueryEnv client")
queryenv = QueryEnvService(queryenv_url, server_id,
cnf.key_path(cnf.DEFAULT_KEY), '2008-12-16')
queryenv_latest = queryenv.get_latest_version()
queryenv = QueryEnvService(queryenv_url, server_id,
cnf.key_path(cnf.DEFAULT_KEY),
queryenv_latest)
if tuple(map(int, queryenv_latest.split('-'))) >= (2012, 7, 1):
scalr_version = queryenv.get_global_config()['params'].get(
'scalr.version')
if scalr_version:
bus.scalr_version = tuple(map(int, scalr_version.split('.')))
version_file = cnf.private_path('.scalr-version')
with open(version_file, 'w') as fp:
fp.write(scalr_version)
bus.queryenv_service = queryenv
bus.queryenv_version = tuple(map(int, queryenv.api_version.split('-')))
if __node__['state'] != 'importing':
lfrp = bus.queryenv_service.list_farm_role_params(
__node__['farm_role_id'])['params']
__node__['base'].update(lfrp.get('base', {}))
ports_non_default = self._select_control_ports()
logger.debug("Initialize messaging")
factory = MessageServiceFactory()
try:
params = dict(ini.items("messaging_" + messaging_adp))
if ports_non_default:
consumer_url = list(
urlparse(params[P2pConfigOptions.CONSUMER_URL]))
consumer_url[1] = ':'.join(
(consumer_url[1].split(':')[0],
str(__node__['base']['messaging_port'])))
params[P2pConfigOptions.CONSUMER_URL] = urlunparse(
consumer_url)
params[P2pConfigOptions.SERVER_ID] = server_id
params[P2pConfigOptions.CRYPTO_KEY_PATH] = cnf.key_path(
cnf.DEFAULT_KEY)
msg_service = factory.new_service(messaging_adp, **params)
bus.messaging_service = msg_service
except (BaseException, Exception):
raise ScalarizrError(
"Cannot create messaging service adapter '%s'" %
(messaging_adp))
optparser = bus.optparser
if optparser and not optparser.values.import_server and linux.os[
'family'] != 'Windows':
system_packages = pkgmgr.package_mgr().list()
for behavior in __node__['behavior']:
if behavior in ['base', 'mongodb'
] or behavior not in api.api_routes.keys():
continue
try:
api_cls = util.import_class(api.api_routes[behavior])
api_cls.check_software(system_packages)
except exceptions.NotFound as e:
logger.error(e)
except exceptions.UnsupportedBehavior as e:
if e.args[0] == 'chef':
# We pass it, cause a lot of roles has chef behavior without chef installed on them
continue
__node__['messaging'].send('RuntimeError',
body={
'code':
'UnsupportedBehavior',
'message': str(e)
})
raise sys.exc_info()[0], sys.exc_info()[1], sys.exc_info(
)[2]
logger.debug('Initialize message handlers')
consumer = msg_service.get_consumer()
consumer.listeners.append(MessageListener())
producer = msg_service.get_producer()
def msg_meta(queue, message):
"""
Add scalarizr version to meta
"""
message.meta.update({
'szr_version':
__version__,
'timestamp':
os_time.utcnow().strftime("%a %d %b %Y %H:%M:%S %z")
})
producer.on('before_send', msg_meta)
Storage.maintain_volume_table = True
if not bus.api_server:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
api_port = 8010
try:
sock.connect(('0.0.0.0', api_port))
api_port = 8009
sock.close()
except socket.error:
pass
STATE['global.api_port'] = api_port
api_app = jsonrpc_http.WsgiApplication(
rpc.RequestHandler(api.api_routes),
cnf.key_path(cnf.DEFAULT_KEY))
class ThreadingWSGIServer(SocketServer.ThreadingMixIn,
wsgiref.simple_server.WSGIServer):
pass
bus.api_server = wsgiref.simple_server.make_server(
'0.0.0.0',
__node__['base']['api_port'],
api_app,
server_class=ThreadingWSGIServer)
if ports_non_default:
msg = msg_service.new_message(
'HostUpdate', None, {
'base': {
'api_port': __node__['base']['api_port'],
'messaging_port': __node__['base']['messaging_port']
}
})
msg_service.get_producer().send(Queues.CONTROL, msg)
def _init_services(self):
logger = logging.getLogger(__name__)
cnf = bus.cnf; ini = cnf.rawini
server_id = ini.get('general', 'server_id')
queryenv_url = ini.get('general', 'queryenv_url')
messaging_adp = ini.get('messaging', 'adapter')
# Set base URL
pr = urlparse(queryenv_url)
bus.scalr_url = urlunparse((pr.scheme, pr.netloc, '', '', '', ''))
logger.debug("Got scalr url: '%s'" % bus.scalr_url)
if not linux.os.windows and node.__node__['platform'].name in ('eucalyptus', 'openstack'):
self._try_resolver(bus.scalr_url)
# Create periodical executor for background tasks (cleanup, rotate, gc, etc...)
bus.periodical_executor = PeriodicalExecutor()
logger.debug("Initialize QueryEnv client")
queryenv = QueryEnvService(queryenv_url, server_id, cnf.key_path(cnf.DEFAULT_KEY), '2008-12-16')
queryenv_latest = queryenv.get_latest_version()
queryenv = QueryEnvService(queryenv_url, server_id, cnf.key_path(cnf.DEFAULT_KEY), queryenv_latest)
if tuple(map(int, queryenv_latest.split('-'))) >= (2012, 7, 1):
scalr_version = queryenv.get_global_config()['params'].get('scalr.version')
if scalr_version:
bus.scalr_version = tuple(map(int, scalr_version.split('.')))
version_file = cnf.private_path('.scalr-version')
with open(version_file, 'w') as fp:
fp.write(scalr_version)
bus.queryenv_service = queryenv
bus.queryenv_version = tuple(map(int, queryenv.api_version.split('-')))
ports_non_default = self._select_control_ports()
logger.debug("Initialize messaging")
factory = MessageServiceFactory()
try:
params = dict(ini.items("messaging_" + messaging_adp))
if ports_non_default:
consumer_url = list(urlparse(params[P2pConfigOptions.CONSUMER_URL]))
consumer_url[1] = ':'.join((consumer_url[1].split(':')[0], str(node.__node__['base']['messaging_port'])))
params[P2pConfigOptions.CONSUMER_URL] = urlunparse(consumer_url)
params[P2pConfigOptions.SERVER_ID] = server_id
params[P2pConfigOptions.CRYPTO_KEY_PATH] = cnf.key_path(cnf.DEFAULT_KEY)
msg_service = factory.new_service(messaging_adp, **params)
bus.messaging_service = msg_service
except (BaseException, Exception):
raise ScalarizrError("Cannot create messaging service adapter '%s'" % (messaging_adp))
logger.debug('Initialize message handlers')
consumer = msg_service.get_consumer()
consumer.listeners.append(MessageListener())
if not linux.os.windows_family:
logger.debug('Schedule SNMP process')
self._snmp_scheduled_start_time = time.time()
Storage.maintain_volume_table = True
if not bus.api_server:
api_app = jsonrpc_http.WsgiApplication(rpc.RequestHandler(_api_routes),
cnf.key_path(cnf.DEFAULT_KEY))
class ThreadingWSGIServer(SocketServer.ThreadingMixIn, wsgiref.simple_server.WSGIServer):
pass
bus.api_server = wsgiref.simple_server.make_server('0.0.0.0',
node.__node__['base']['api_port'],
api_app,
server_class=ThreadingWSGIServer)
if ports_non_default:
msg = msg_service.new_message('HostUpdate', None, {
'base': {
'api_port': node.__node__['base']['api_port'],
'messaging_port': node.__node__['base']['messaging_port']
}
})
msg_service.get_producer().send(Queues.CONTROL, msg)
def _init_services(self):
logger = logging.getLogger(__name__)
cnf = bus.cnf; ini = cnf.rawini
server_id = ini.get('general', 'server_id')
queryenv_url = ini.get('general', 'queryenv_url')
messaging_adp = ini.get('messaging', 'adapter')
# Set base URL
pr = urlparse(queryenv_url)
bus.scalr_url = urlunparse((pr.scheme, pr.netloc, '', '', '', ''))
logger.debug("Got scalr url: '%s'" % bus.scalr_url)
# Create periodical executor for background tasks (cleanup, rotate, gc, etc...)
bus.periodical_executor = PeriodicalExecutor()
logger.debug("Initialize QueryEnv client")
queryenv = QueryEnvService(queryenv_url, server_id, cnf.key_path(cnf.DEFAULT_KEY), '2008-12-16')
queryenv_latest = queryenv.get_latest_version()
queryenv = QueryEnvService(queryenv_url, server_id, cnf.key_path(cnf.DEFAULT_KEY), queryenv_latest)
if tuple(map(int, queryenv_latest.split('-'))) >= (2012, 7, 1):
scalr_version = queryenv.get_global_config()['params'].get('scalr.version')
if scalr_version:
bus.scalr_version = tuple(map(int, scalr_version.split('.')))
version_file = cnf.private_path('.scalr-version')
with open(version_file, 'w') as fp:
fp.write(scalr_version)
bus.queryenv_service = queryenv
bus.queryenv_version = tuple(map(int, queryenv.api_version.split('-')))
logger.debug("Initialize messaging")
factory = MessageServiceFactory()
try:
params = dict(ini.items("messaging_" + messaging_adp))
params[P2pConfigOptions.SERVER_ID] = server_id
params[P2pConfigOptions.CRYPTO_KEY_PATH] = cnf.key_path(cnf.DEFAULT_KEY)
msg_service = factory.new_service(messaging_adp, **params)
bus.messaging_service = msg_service
except (BaseException, Exception):
raise ScalarizrError("Cannot create messaging service adapter '%s'" % (messaging_adp))
logger.debug('Initialize message handlers')
consumer = msg_service.get_consumer()
consumer.listeners.append(MessageListener())
if not linux.os.windows_family:
logger.debug('Schedule SNMP process')
self._snmp_scheduled_start_time = time.time()
Storage.maintain_volume_table = True
if not bus.api_server:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
api_port = 8010
try:
sock.connect(('0.0.0.0', api_port))
api_port = 8009
sock.close()
except socket.error:
pass
STATE['global.api_port'] = api_port
api_app = jsonrpc_http.WsgiApplication(rpc.RequestHandler(_api_routes),
cnf.key_path(cnf.DEFAULT_KEY))
class ThreadingWSGIServer(SocketServer.ThreadingMixIn, wsgiref.simple_server.WSGIServer):
pass
bus.api_server = wsgiref.simple_server.make_server('0.0.0.0',
api_port, api_app, server_class=ThreadingWSGIServer)
def _init_services(self):
logger = logging.getLogger(__name__)
cnf = bus.cnf; ini = cnf.rawini
server_id = ini.get('general', 'server_id')
queryenv_url = ini.get('general', 'queryenv_url')
messaging_adp = ini.get('messaging', 'adapter')
# Set base URL
pr = urlparse(queryenv_url)
bus.scalr_url = urlunparse((pr.scheme, pr.netloc, '', '', '', ''))
logger.debug("Got scalr url: '%s'" % bus.scalr_url)
if not linux.os.windows and __node__['platform'].name == 'openstack':
self._try_resolver(bus.scalr_url)
# Create periodical executor for background tasks (cleanup, rotate, gc, etc...)
bus.periodical_executor = PeriodicalExecutor()
logger.debug("Initialize QueryEnv client")
queryenv = QueryEnvService(queryenv_url, server_id, cnf.key_path(cnf.DEFAULT_KEY), '2008-12-16')
queryenv_latest = queryenv.get_latest_version()
queryenv = QueryEnvService(queryenv_url, server_id, cnf.key_path(cnf.DEFAULT_KEY), queryenv_latest)
if tuple(map(int, queryenv_latest.split('-'))) >= (2012, 7, 1):
scalr_version = queryenv.get_global_config()['params'].get('scalr.version')
if scalr_version:
bus.scalr_version = tuple(map(int, scalr_version.split('.')))
version_file = cnf.private_path('.scalr-version')
with open(version_file, 'w') as fp:
fp.write(scalr_version)
bus.queryenv_service = queryenv
bus.queryenv_version = tuple(map(int, queryenv.api_version.split('-')))
if __node__['state'] != 'importing':
lfrp = bus.queryenv_service.list_farm_role_params(__node__['farm_role_id'])['params']
__node__['base'].update(lfrp.get('base', {}))
ports_non_default = self._select_control_ports()
logger.debug("Initialize messaging")
factory = MessageServiceFactory()
try:
params = dict(ini.items("messaging_" + messaging_adp))
if ports_non_default:
consumer_url = list(urlparse(params[P2pConfigOptions.CONSUMER_URL]))
consumer_url[1] = ':'.join((consumer_url[1].split(':')[0], str(__node__['base']['messaging_port'])))
params[P2pConfigOptions.CONSUMER_URL] = urlunparse(consumer_url)
params[P2pConfigOptions.SERVER_ID] = server_id
params[P2pConfigOptions.CRYPTO_KEY_PATH] = cnf.key_path(cnf.DEFAULT_KEY)
msg_service = factory.new_service(messaging_adp, **params)
bus.messaging_service = msg_service
except (BaseException, Exception):
raise ScalarizrError("Cannot create messaging service adapter '%s'" % (messaging_adp))
optparser = bus.optparser
if optparser and not optparser.values.import_server and linux.os['family'] != 'Windows':
system_packages = pkgmgr.package_mgr().list()
for behavior in __node__['behavior']:
if behavior in ['base', 'mongodb'] or behavior not in api.api_routes.keys():
continue
try:
api_cls = util.import_class(api.api_routes[behavior])
api_cls.check_software(system_packages)
except exceptions.NotFound as e:
logger.error(e)
except exceptions.UnsupportedBehavior as e:
if e.args[0] == 'chef':
# We pass it, cause a lot of roles has chef behavior without chef installed on them
continue
__node__['messaging'].send(
'RuntimeError',
body={
'code': 'UnsupportedBehavior',
'message': str(e)
}
)
raise sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]
logger.debug('Initialize message handlers')
consumer = msg_service.get_consumer()
consumer.listeners.append(MessageListener())
producer = msg_service.get_producer()
def msg_meta(queue, message):
"""
Add scalarizr version to meta
"""
message.meta.update({
'szr_version': __version__,
'timestamp': os_time.utcnow().strftime("%a %d %b %Y %H:%M:%S %z")
})
producer.on('before_send', msg_meta)
Storage.maintain_volume_table = True
if not bus.api_server:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
api_port = 8010
try:
sock.connect(('0.0.0.0', api_port))
api_port = 8009
sock.close()
except socket.error:
pass
STATE['global.api_port'] = api_port
api_app = jsonrpc_http.WsgiApplication(rpc.RequestHandler(api.api_routes),
cnf.key_path(cnf.DEFAULT_KEY))
class ThreadingWSGIServer(SocketServer.ThreadingMixIn, wsgiref.simple_server.WSGIServer):
pass
bus.api_server = wsgiref.simple_server.make_server('0.0.0.0',
__node__['base']['api_port'],
api_app,
server_class=ThreadingWSGIServer)
if ports_non_default:
msg = msg_service.new_message('HostUpdate', None, {
'base': {
'api_port': __node__['base']['api_port'],
'messaging_port': __node__['base']['messaging_port']
}
})
msg_service.get_producer().send(Queues.CONTROL, msg)