def auth(self):
status = True
try:
self.api = Pepper(self.metadata['auth_url'])
self.api.login(self.metadata['username'],
self.metadata['password'], 'pam')
except PepperException as exception:
logger.error(exception)
status = False
except URLError as exception:
logger.error(exception)
status = False
return status
def __init__(self, **kwargs):
self.kind = 'salt'
super(SaltStackInput, self).__init__(**kwargs)
try:
self.name = kwargs['name']
except KeyError:
raise ValueError('Missing parameter name')
config_data = load_yaml_json_file(kwargs['config_file'])
self.config = config_data['configs'][self.name]
self.api = Pepper(self.config['url'])
self.api.login(self.config['auth']['username'],
self.config['auth']['password'], 'pam')
def api(self):
def login():
LOG.info("Authentication in Salt API")
self.__api.login(username=self.__user,
password=self.__password,
eauth='pam')
return datetime.now()
if self.__api:
if (datetime.now() - self.__session_start).seconds < 5 * 60:
return self.__api
else:
# FIXXME: Change to debug
LOG.info("Session's expired")
self.__session_start = login()
return self.__api
url = "http://{host}:{port}".format(host=self.host, port=self.port)
LOG.info("Connecting to Salt API {0}".format(url))
self.__api = Pepper(url)
self.__session_start = login()
return self.__api
class SaltStackInput(BaseInput):
RESOURCE_MAP = {
'salt_high_state': {
'resource': 'High State',
'icon': 'fa:cube',
},
'salt_job': {
'resource': 'Job',
'icon': 'fa:clock-o',
},
'salt_low_state': {
'resource': 'Low State',
'icon': 'fa:cube',
},
'salt_minion': {
'resource': 'Minion',
'icon': 'fa:server',
},
'salt_service': {
'resource': 'Service',
'icon': 'fa:podcast',
},
'salt_state_module': {
'resource': 'State Module',
'icon': 'fa:cubes',
},
'salt_user': {
'resource': 'User',
'icon': 'fa:user',
},
}
def __init__(self, **kwargs):
self.kind = 'salt'
super(SaltStackInput, self).__init__(**kwargs)
try:
self.name = kwargs['name']
except KeyError:
raise ValueError('Missing parameter name')
config_data = load_yaml_json_file(kwargs['config_file'])
self.config = config_data['configs'][self.name]
self.api = Pepper(self.config['url'])
self.api.login(self.config['auth']['username'],
self.config['auth']['password'], 'pam')
def scrape_all_resources(self):
self.scrape_jobs()
self.scrape_minions()
self.scrape_services()
self.scrape_high_states()
# self.scrape_low_states()
def _create_relations(self):
"""
for resource_id, resource in self.resources['salt_low_state'].items():
# Define relationships between low states and nodes.
self._scrape_relation(
'salt_minion-salt_low_state',
resource['metadata']['minion'],
resource_id)
split_service = resource['metadata']['__sls__'].split('.')
self._scrape_relation(
'salt_service-salt_low_state',
'{}|{}.{}'.format(resource['metadata']['minion'],
split_service[0], split_service[1]),
resource_id)
"""
for resource_id, resource in self.resources.get('salt_high_state',
{}).items():
# Define relationships between high states and nodes.
self._scrape_relation('salt_minion-salt_high_state',
resource['metadata']['minion'], resource_id)
split_service = resource['metadata']['__sls__'].split('.')
self._scrape_relation(
'salt_service-salt_high_state',
'{}|{}.{}'.format(resource['metadata']['minion'],
split_service[0], split_service[1]),
resource_id)
for resource_id, resource in self.resources.get('salt_service',
{}).items():
self._scrape_relation('salt_service-salt_minion', resource_id,
resource['metadata']['host'])
for resource_id, resource in self.resources.get('salt_job',
{}).items():
self._scrape_relation('salt_user-salt_job',
resource['metadata']['User'], resource_id)
for minion_id, result in resource['metadata'].get('Result',
{}).items():
self._scrape_relation('salt_job-salt_minion', resource_id,
minion_id)
if type(result) is list:
logger.error(result[0])
else:
for state_id, state in result.items():
if '__id__' in state:
result_id = '{}|{}'.format(minion_id,
state['__id__'])
self._scrape_relation('salt_job-salt_high_state',
resource_id, result_id)
def scrape_jobs(self):
response = self.api.low([{
'client':
'runner',
'fun':
'jobs.list_jobs',
'arg':
"search_function='[\"state.apply\", \"state.sls\"]'"
}]).get('return')[0]
for job_id, job in response.items():
if job['Function'] in ['state.apply', 'state.sls']:
result = self.api.lookup_jid(job_id).get('return')[0]
job['Result'] = result
self._scrape_resource(job_id,
job['Function'],
'salt_job',
None,
metadata=job)
self._scrape_resource(job['User'],
job['User'],
'salt_user',
None,
metadata={})
def scrape_minions(self):
response = self.api.low([{
'client': 'local',
'tgt': '*',
'fun': 'grains.items'
}]).get('return')[0]
for minion_id, minion in response.items():
self._scrape_resource(minion_id,
minion_id,
'salt_minion',
None,
metadata=minion)
def scrape_services(self):
response = self.api.low([{
'client': 'local',
'expr_form': 'compound',
'tgt': 'I@salt:master',
'fun': 'saltresource.graph_data'
}]).get('return')[0]
for minion_id, minion in response.items():
for service in minion['graph']:
self._scrape_resource('{}|{}'.format(minion_id,
service['service']),
service['service'],
'salt_service',
None,
metadata=service)
def scrape_low_states(self):
response = self.api.low([{
'client': 'local',
'tgt': '*',
'fun': 'state.show_lowstate'
}]).get('return')[0]
for minion_id, low_states in response.items():
for low_state in low_states:
low_state['minion'] = minion_id
self._scrape_resource('{}|{}|{}'.format(
minion_id, low_state['state'], low_state['__id__']),
'{} {}'.format(low_state['state'],
low_state['__id__']),
'salt_low_state',
None,
metadata=low_state)
def scrape_high_states(self):
response = self.api.low([{
'client': 'local',
'tgt': '*',
'fun': 'state.show_highstate'
}]).get('return')[0]
for minion_id, high_states in response.items():
if type(high_states) is list:
logger.error(high_states[0])
else:
for high_state_id, high_state in high_states.items():
high_state['minion'] = minion_id
self._scrape_resource('{}|{}'.format(
minion_id, high_state_id),
high_state_id,
'salt_high_state',
None,
metadata=high_state)
class SaltStackClient(BaseClient):
def __init__(self, **kwargs):
super(SaltStackClient, self).__init__(**kwargs)
def auth(self):
status = True
try:
self.api = Pepper(self.metadata['auth_url'])
self.api.login(self.metadata['username'],
self.metadata['password'], 'pam')
except PepperException as exception:
logger.error(exception)
status = False
except URLError as exception:
logger.error(exception)
status = False
return status
def update_resources(self, resources=None):
if self.auth():
if resources is None:
resources = DEFAULT_RESOURCES
for resource in resources:
metadata = self.get_resource_metadata(resource)
self.process_resource_metadata(resource, metadata)
count = len(self.resources.get(resource, {}))
logger.info("Processed {} {} resources".format(
count, resource))
self.process_relation_metadata()
def get_resource_status(self, kind, metadata):
if not isinstance(metadata, dict):
return 'unknown'
if kind == 'salt_minion':
if 'id' in metadata:
return 'active'
return 'unknown'
def get_resource_metadata(self, kind):
logger.info("Getting {} resources".format(kind))
if kind == 'salt_job':
metadata = self.api.low([{
'client': 'runner',
'fun': 'jobs.list_jobs',
'arg': "search_function='[\"state.apply\", \"state.sls\"]'",
'timeout': 60
}]).get('return')[0]
elif kind == 'salt_lowstate':
metadata = self.api.low([{
'client': 'local',
'tgt': '*',
'fun': 'state.show_lowstate',
'timeout': 60
}]).get('return')[0]
elif kind == 'salt_minion':
metadata = self.api.low([{
'client': 'local',
'tgt': '*',
'fun': 'grains.items',
'timeout': 60
}]).get('return')[0]
elif kind == 'salt_service':
metadata = self.api.low([{
'client': 'local',
'tgt': '*',
'fun': 'pillar.data',
'timeout': 60
}]).get('return')[0]
else:
metadata = {}
if not isinstance(metadata, dict):
metadata = {}
return metadata
def process_resource_metadata(self, kind, metadata):
if kind == 'salt_event':
manager = Manager.objects.get(name=metadata.get('manager'))
roles = []
if isinstance(metadata.get('return'), (list, tuple)):
return
for datum_name, datum in metadata.get('return', {}).items():
try:
uid = '{}|{}'.format(metadata['id'], datum['__id__'])
except KeyError as exception:
logger.error('No key {} in {}'.format(exception, datum))
continue
try:
lowstate = Resource.objects.get(uid=uid, manager=manager)
except Resource.DoesNotExist:
logger.error('No salt_lowstate resource '
'with UID {} found'.format(uid))
continue
to_save = False
if 'apply' not in lowstate.metadata:
lowstate.metadata['apply'] = {}
lowstate.metadata['apply'][metadata.get('jid')] = datum
if datum['result']:
lowstate.status = 'active'
else:
lowstate.status = 'error'
to_save = True
else:
if metadata.get('jid') not in lowstate.metadata['apply']:
lowstate.metadata['apply'][metadata.get('jid')] = datum
if datum['result']:
lowstate.status = 'active'
else:
lowstate.status = 'error'
to_save = True
if to_save:
lowstate.save()
role_parts = lowstate.metadata['__sls__'].split('.')
role_name = "{}-{}".format(role_parts[0], role_parts[1])
roles.append(role_name)
for role_name in set(roles):
uid = '{}|{}'.format(metadata['id'], role_name)
try:
service = Resource.objects.get(uid=uid, manager=manager)
except Resource.DoesNotExist:
logger.error('No salt_service resource '
'with UID {} found'.format(uid))
continue
errors = 0
unknown = 0
to_save = False
lowstate_links = service.source.filter(kind='state_of_service')
for lowstate_link in lowstate_links:
if lowstate_link.target.status == 'error':
errors += 1
elif lowstate_link.target.status == 'unknown':
unknown += 1
if errors > 0 and service.status != 'error':
service.status = 'error'
to_save = True
if unknown > 0 and service.status != 'unknown':
service.status = 'build'
to_save = True
elif service.status != 'active':
service.status = 'active'
to_save = True
if to_save:
service.save()
elif kind == 'salt_job':
for job_id, job in metadata.items():
if not isinstance(job, dict):
continue
if job['Function'] in ['state.apply', 'state.sls']:
result = self.api.lookup_jid(job_id).get('return')[0]
job['Result'] = result
self._create_resource(job_id,
job['Function'],
'salt_job',
metadata=job)
self._create_resource(job['User'],
job['User'].replace('sudo_', ''),
'salt_user',
metadata={})
elif kind == 'salt_lowstate':
for minion_id, low_states in metadata.items():
if not isinstance(low_states, list):
continue
for low_state in low_states:
if not isinstance(low_state, dict):
logger.error('Salt lowtate {} parsing problem on '
'{}'.format(low_state, minion_id))
continue
low_state['minion'] = minion_id
self._create_resource(
'{}|{}'.format(minion_id, low_state['__id__']),
'{} {}'.format(low_state['state'],
low_state['__id__']),
'salt_lowstate',
metadata=low_state)
elif kind == 'salt_minion':
self._create_resource('salt-master',
'salt-master',
'salt_master',
metadata={})
for minion_id, minion_data in metadata.items():
self._create_resource(minion_id,
minion_id,
'salt_minion',
metadata={'grains': minion_data})
elif kind == 'salt_service':
for minion_id, minion_data in metadata.items():
if not isinstance(minion_data, dict):
continue
for service_name, service in minion_data.items():
if service_name not in settings.RECLASS_SERVICE_BLACKLIST:
if not isinstance(service, dict):
logger.error('Salt service {} parsing problem: '
'{} on {}'.format(
service_name, service, minion_id))
continue
for role_name, role in service.items():
if role_name not in settings.RECLASS_ROLE_BLACKLIST:
service_key = '{}-{}'.format(
service_name, role_name)
self._create_resource('{}|{}'.format(
minion_id, service_key),
service_key,
'salt_service',
metadata={
'pillar': {
service_name: {
role_name:
role
}
}
})
def process_relation_metadata(self):
# Define relationships between minions and master
for resource_id, resource in self.resources.get('salt_minion',
{}).items():
self._create_relation('controlled_by_master', resource_id,
'salt-master')
# Define relationships between services and minions
for resource_id, resource in self.resources.get('salt_service',
{}).items():
self._create_relation('runs_on_minion', resource_id,
resource_id.split('|')[0])
# Define relationships between lowstates and services
for resource_id, resource in self.resources.get('salt_lowstate',
{}).items():
split_service = resource['metadata']['__sls__'].split('.')
self._create_relation(
'state_of_service', resource_id,
'{}|{}-{}'.format(resource['metadata']['minion'],
split_service[0], split_service[1]))
for resource_id, resource in self.resources.get('salt_job',
{}).items():
self._create_relation('action_by_user', resource_id,
resource['metadata']['User'])
for minion_id, result in resource['metadata'].get('Result',
{}).items():
self._create_relation('applied_on_minion', resource_id,
minion_id)
if type(result) is list:
logger.error(result[0])
else:
for state_id, state in result.items():
if '__id__' in state:
result_id = '{}|{}'.format(minion_id,
state['__id__'])
self._create_relation('applied_lowstate',
resource_id, result_id)
def get_resource_action_fields(self, resource, action):
fields = {}
if resource.kind == 'salt_minion':
if action == 'run_module':
fields['function'] = forms.CharField(label='Module function',
initial='cmd.run')
fields['arguments'] = forms.CharField(
widget=forms.Textarea(attrs={
'rows': 3,
'cols': 40
}),
required=False,
label='Function arguments')
return fields
def process_resource_action(self, resource, action, data):
if resource.kind == 'salt_minion':
if action == 'run_module':
if self.auth():
run_metadata = {
'client': 'local',
'tgt': resource.uid,
'fun': data['function'],
'timeout': 60
}
if data['arguments'] != '':
run_metadata['arg'] = [
s.strip() for s in data['arguments'].splitlines()
]
metadata = self.api.low([run_metadata]).get('return')[0]
logger.info(metadata)
def __init__(self):
self.authenticated = False
self.api = Pepper(os.environ.get("SALT_HOST"))
self.api_auth()
class PepperApi():
def __init__(self):
self.authenticated = False
self.api = Pepper(os.environ.get("SALT_HOST"))
self.api_auth()
def api_auth(self):
try:
self.api.login(os.environ.get("SALT_USERNAME"),
os.environ.get("SALT_SHARED_SECRET"),
'sharedsecret')
self.authenticated = True
except Exception as err:
logger.error("Error authing to salt: {0}".format(err))
self.authenticated = False
def salt_keys(self):
"""
Queries the Salt Master for all Minion keys
Returns:
dict: A dictionary with 3 lists for minions,
pre-minions and rejected minions
"""
# check auth state
self.api_auth()
api_reponse = self.api.low([{
'client': 'wheel',
'fun': 'key.list_all'
}])
if api_reponse['return'][0]['data']['success']:
return api_reponse['return'][0]['data']['return']
def accept_key(self, minion_id):
"""
Tells the Salt Master to Accept a Key by its name
Args:
minion_id (str): Minion ID -
should be a Mongo ID from `Hive` modal
Returns:
dict: A dictionary with the ID for any accepted minions.
"""
# check auth state
self.api_auth()
api_reponse = self.api.low([{
'client': 'wheel',
'fun': 'key.accept',
'match': minion_id
}])
if api_reponse['return'][0]['data']['success']:
return api_reponse['return'][0]['data']['return']
def delete_key(self, minion_id):
"""Given a Minion ID will delete the key from the salt master"""
# check auth state
self.api_auth()
api_reponse = self.api.low([{
'client': 'wheel',
'fun': 'key.delete',
'match': minion_id
}])
if api_reponse['return'][0]['data']['success']:
return api_reponse['return'][0]['data']['return']
def run_client_function_async(self, target, function, arg_list=[]):
"""
Basic Function Runner; for things like test.ping it is up to
the receiving function to parse the data it needs.
Makes Async Call returns Job ID
"""
# check auth state
self.api_auth()
api_reponse = self.api.low([{
'client': 'local_async',
'tgt': target,
'fun': function,
'arg': arg_list
}])
return api_reponse['return'][0]['jid']
def run_client_function(self, target, function, arg_list=[]):
"""
Basic Function Runner; for things like test.ping it
is up to the receiving function to parse the data it needs.
Not async so blocking?
"""
# check auth state
self.api_auth()
api_reponse = self.api.low([{
'client': 'local',
'tgt': target,
'fun': function,
'arg': arg_list
}])
return api_reponse['return'][0]
def apply_state(self, target, args_list):
"""
Tells the Salt Master to apply a given state to a named minion
Args:
target (str): Minion ID - should be a Mongo ID
from `Hive` modal
args_list (list): List of args. First element in list MUST
be the Salt State to apply
Returns:
str: Salt Job ID
"""
# check auth state
self.api_auth()
api_reponse = self.api.low([{
'client': 'local_async',
'tgt': target,
'fun': "state.apply",
'arg': args_list
}])
return api_reponse['return'][0]['jid']
def lookup_job(self, job_id):
"""
Query the Salt Master for a job by its Salt Job ID
Args:
job_id (str): Salt Job Id should be taken from PepperJobs Model
Returns:
dict/None: Return None if job still pending else dictionary of
all state modules and their outputs.
"""
# check auth state
self.api_auth()
try:
api_response = self.api.lookup_jid(job_id)
if len(api_response['return'][0]) == 0:
return None
else:
return api_response['return'][0]
except Exception as err:
logger.error("Error authing to salt: {0}".format(err))
return None
def docker_state(self, target, container):
"""
Get the status of a docker container.
"""
# check auth state
self.api_auth()
api_reponse = self.api.low([{
'client': 'local_async',
'tgt': target,
'fun': 'docker.state',
'arg': container
}])
return api_reponse['return'][0]['jid']
def docker_control(self, target, container, wanted_state):
"""
Set the status of a container.
The container must already exist
"""
# check auth state
self.api_auth()
if wanted_state == "start":
function = "docker.start"
elif wanted_state == "stop":
function = "docker.stop"
api_reponse = self.api.low([{
'client': 'local_async',
'tgt': target,
'fun': function,
'arg': container
}])
return api_reponse['return'][0]['jid']
def docker_remove(self, target, container):
"""
Remove a container and destroy its image
"""
# check auth state
self.api_auth()
responses = []
# Stop the container
api_reponse = self.api.low([{
'client': 'local',
'tgt': target,
'fun': 'docker.stop',
'arg': container
}])
responses.append(api_reponse)
# Remove the container
api_reponse = self.api.low([{
'client': 'local',
'tgt': target,
'fun': 'docker.rm',
'arg': container
}])
responses.append(api_reponse)
# Check state to confirm
api_reponse = self.api.low([{
'client': 'local',
'tgt': target,
'fun': 'docker.state',
'arg': container
}])
try:
result_object = api_reponse['return'][0]
if 'ERROR' in result_object[target]:
return True
else:
return False
except Exception as err:
logging.error(err)
return False
def __init__(self, **kwargs):
self.kind = 'salt'
super(SaltStackInput, self).__init__(**kwargs)
self.api = Pepper(self.config['auth_url'])
self.api.login(self.config['username'], self.config['password'], 'pam')
class SaltStackInput(BaseInput):
def __init__(self, **kwargs):
self.kind = 'salt'
super(SaltStackInput, self).__init__(**kwargs)
self.api = Pepper(self.config['auth_url'])
self.api.login(self.config['username'], self.config['password'], 'pam')
def scrape_all_resources(self):
self.scrape_jobs()
self.scrape_minions()
self.scrape_services()
self.scrape_high_states()
# self.scrape_low_states()
def _create_relations(self):
"""
for resource_id, resource in self.resources['salt_low_state'].items():
# Define relationships between low states and nodes.
self._scrape_relation(
'salt_minion-salt_low_state',
resource['metadata']['minion'],
resource_id)
split_service = resource['metadata']['__sls__'].split('.')
self._scrape_relation(
'salt_service-salt_low_state',
'{}|{}.{}'.format(resource['metadata']['minion'],
split_service[0], split_service[1]),
resource_id)
"""
for resource_id, resource in self.resources.get('salt_high_state',
{}).items():
# Define relationships between high states and nodes.
self._scrape_relation('on_salt_minion', resource_id,
resource['metadata']['minion'])
split_service = resource['metadata']['__sls__'].split('.')
self._scrape_relation(
'contains_salt_high_state',
'{}|{}.{}'.format(resource['metadata']['minion'],
split_service[0], split_service[1]),
resource_id)
for resource_id, resource in self.resources.get('salt_service',
{}).items():
self._scrape_relation('on_salt_minion', resource_id,
resource['metadata']['host'])
for resource_id, resource in self.resources.get('salt_job',
{}).items():
self._scrape_relation('by_salt_user', resource_id,
resource['metadata']['User'])
for minion_id, result in resource['metadata'].get('Result',
{}).items():
self._scrape_relation('on_salt_minion', resource_id, minion_id)
if type(result) is list:
logger.error(result[0])
else:
for state_id, state in result.items():
if '__id__' in state:
result_id = '{}|{}'.format(minion_id,
state['__id__'])
self._scrape_relation('contains_salt_high_state',
resource_id, result_id)
def scrape_jobs(self):
response = self.api.low([{
'client':
'runner',
'fun':
'jobs.list_jobs',
'arg':
"search_function='[\"state.apply\", \"state.sls\"]'"
}]).get('return')[0]
for job_id, job in response.items():
if job['Function'] in ['state.apply', 'state.sls']:
result = self.api.lookup_jid(job_id).get('return')[0]
job['Result'] = result
self._scrape_resource(job_id,
job['Function'],
'salt_job',
None,
metadata=job)
self._scrape_resource(job['User'],
job['User'],
'salt_user',
None,
metadata={})
def scrape_minions(self):
response = self.api.low([{
'client': 'local',
'tgt': '*',
'fun': 'grains.items'
}]).get('return')[0]
for minion_id, minion in response.items():
self._scrape_resource(minion_id,
minion_id,
'salt_minion',
None,
metadata=minion)
def scrape_services(self):
response = self.api.low([{
'client': 'local',
'expr_form': 'compound',
'tgt': 'I@salt:master',
'fun': 'saltresource.graph_data'
}]).get('return')[0]
for minion_id, minion in response.items():
for service in minion['graph']:
self._scrape_resource('{}|{}'.format(minion_id,
service['service']),
service['service'],
'salt_service',
None,
metadata=service)
def scrape_low_states(self):
response = self.api.low([{
'client': 'local',
'tgt': '*',
'fun': 'state.show_lowstate'
}]).get('return')[0]
for minion_id, low_states in response.items():
for low_state in low_states:
low_state['minion'] = minion_id
self._scrape_resource('{}|{}|{}'.format(
minion_id, low_state['state'], low_state['__id__']),
'{} {}'.format(low_state['state'],
low_state['__id__']),
'salt_low_state',
None,
metadata=low_state)
def scrape_high_states(self):
response = self.api.low([{
'client': 'local',
'tgt': '*',
'fun': 'state.show_highstate'
}]).get('return')[0]
for minion_id, high_states in response.items():
if type(high_states) is list:
logger.error(high_states[0])
else:
for high_state_id, high_state in high_states.items():
high_state['minion'] = minion_id
self._scrape_resource('{}|{}'.format(
minion_id, high_state_id),
high_state_id,
'salt_high_state',
None,
metadata=high_state)