def delete_files(directory, extension='.yaml'):
"""Delete all files of a specfic extension in a directory.
Args:
directory: Directory name
extension: File extension
Returns:
None
"""
# Determine whether directory is valid
if os.path.isdir(directory) is False:
log_message = ('Directory %s does not exist') % (directory)
log.log2die_safe(1023, log_message)
# Get list of files
filelist = [
next_file for next_file in os.listdir(directory)
if next_file.endswith(extension)
]
# Delete files
for delete_file in filelist:
delete_path = ('%s/%s') % (directory, delete_file)
os.remove(delete_path)
def _python(self):
"""Determine Python version.
Args:
None
Returns:
None
"""
# Initialize key variables
valid = True
major = 3
minor = 5
major_installed = sys.version_info[0]
minor_installed = sys.version_info[1]
# Determine whether python version is too low
if major_installed < major:
valid = False
elif major_installed == major and minor_installed < minor:
valid = False
# Process validity
if valid is False:
log_message = ('Required python version must be >= {}.{}. '
'Python version {}.{} installed'
''.format(major, minor, major_installed,
minor_installed))
log.log2die_safe(1095, log_message)
else:
log_message = ('Python version {}.{}.'
''.format(major_installed, minor_installed))
shared.print_ok(log_message)
def __init__(self):
"""Function for intializing the class.
Args:
None
Returns:
None
"""
# Initialize key variables
self.username = getpass.getuser()
valid = True
major = 3
minor = 5
major_installed = sys.version_info[0]
minor_installed = sys.version_info[1]
# Exit if python version is too low
if major_installed < major:
valid = False
elif major_installed == major and minor_installed < minor:
valid = False
if valid is False:
log_message = ('Required python version must be >= {}.{}. '
'Python version {}.{} installed'
''.format(major, minor, major_installed,
minor_installed))
log.log2die_safe(1135, log_message)
def _install_pip3_packages(self):
"""Install PIP3 packages.
Args:
None
Returns:
None
"""
# Initialize key variables
username = self.username
# Determine whether PIP3 exists
print('Installing required pip3 packages')
pip3 = infoset.utils.general.search_file('pip3')
if pip3 is None:
log_message = ('Cannot find python "pip3". Please install.')
log.log2die_safe(1052, log_message)
# Install required PIP packages
utils_directory = infoset.utils.__path__[0]
requirements_file = ('%s/requirements.txt') % (
Path(utils_directory).parents[1])
if username == 'root':
script_name = ('pip3 install --upgrade --requirement %s'
'') % (requirements_file)
else:
script_name = ('pip3 install --user --upgrade --requirement %s'
'') % (requirements_file)
infoset.utils.general.run_script(script_name)
def run():
"""Do the installation.
Args:
None
Returns:
None
"""
# Initialize key variables
running_username = getpass.getuser()
# Prevent running as sudo user
if 'SUDO_UID' in os.environ:
log_message = (
'Cannot run installation using "sudo". Run as a regular user to '
'install in this directory or as user "root".')
log.log2die_safe(1029, log_message)
# Do specific setups for root user
if running_username != 'root':
_DaemonSystemD().enable()
# Start daemons
_Daemons().start()
# Run the post check
postcheck = _PostCheck()
postcheck.validate()
def _system_daemon_prompt():
"""Prompt user to see whether they want to run as a system daemon.
Args:
None
Returns:
None
"""
# Initialize key variables
running_username = getpass.getuser()
# Return if running as root. We'll run as a daemon
if running_username == 'root':
return
# Get the user's intention
prompt = input('Do you want switchmap-ng to start automatically '
'after a reboot?: (Y, N) ')
intention = prompt.strip()
if bool(intention) is True and len(intention) == 1:
response = intention.lower()[0]
if response == 'n':
return
elif response == 'y':
log_message = ('Run this script as the "root" user to '
'get the automatic functionality.')
log.log2die_safe(1125, log_message)
log_message = 'Please answer "Y" or "N", and try again.'
log.log2die_safe(1128, log_message)
def _systemd(self):
"""Determine if systemd is installed.
Args:
None
Returns:
None
"""
# Initialize key variables.
directory = '/etc/systemd/system'
# Do nothing if this is not the root user.
username = getpass.getuser()
if username != 'root':
return
# Test
if os.path.isdir(directory) is True:
shared.print_ok('Systemd installed.')
else:
log_message = (
'The systemd package isn\'t installed on this system.')
log.log2die_safe(1048, log_message)
def run():
"""Setup infoset-ng.
Args:
None
Returns:
None
"""
# Initialize key variables
username = getpass.getuser()
# Prevent running as sudo user
if 'SUDO_UID' in os.environ:
log_message = (
'Cannot run setup using "sudo". Run as a regular user to '
'install in this directory or as user "root".')
log.log2die_safe(1032, log_message)
# If running as the root user, then the infoset user needs to exist
if username == 'root':
try:
daemon_username = input('Please enter the username under which '
'infoset-ng will run: ')
# Get GID and UID for user
_ = getpwnam(daemon_username).pw_gid
except:
log_message = ('User {} not found. Please try again.'
''.format(daemon_username))
log.log2die_safe(1105, log_message)
else:
daemon_username = username
# Create a configuration only if unittests are not being run
if 'INFOSET_CONFIGDIR' not in os.environ:
config = _ConfigCreate(daemon_username)
config.validate()
config.write()
# Determine whether version of python is valid
_PythonSetupPackages().run()
# Update configuration if required
_ConfigSetup().run()
###########################################################################
# Create database with newly created configuration
# The database libraries cannot be imported if there
# isn't a valid configuration
###########################################################################
executable = 'python3 {}/{}'.format(_MAINT_DIRECTORY, 'database.py')
returncode = os.system(executable)
if bool(returncode) is True:
sys.exit(2)
def _db_connectivity(self):
"""Validate we can connect to the database.
Args:
None
Returns:
None
"""
# Initialize key variables
valid = False
db_hostname = self.config_dict['main']['db_hostname']
db_password = self.config_dict['main']['db_password']
db_username = self.config_dict['main']['db_username']
db_name = self.config_dict['main']['db_name']
# Do test
try:
# Open database connection. Prepare cursor
database = pymysql.connect(host=db_hostname,
user=db_username,
passwd=db_password,
db=db_name)
cursor = database.cursor()
# Do a basic check
cursor.execute('SELECT VERSION()')
results = cursor.fetchone()
# Check result
valid = bool(results)
# disconnect from server
database.close()
except Exception as _:
valid = False
except:
valid = False
# Process validity
if valid is True:
log_message = 'Database connectivity successfully verified.'
shared.print_ok(log_message)
else:
log_message = (
'Cannot connect to the database. Verify your credentials. '
'Database Hostname: "{}", Database Username: "******", '
'Database Name: "{}", Database Password: "******"'
''.format(db_hostname, db_username, db_name))
log.log2die_safe(1067, log_message)
def run(self):
"""Update the configuration with good defaults.
Args:
None
Returns:
None
"""
# Initialize key variables
valid = True
updated_list = []
config = copy.deepcopy(self.config)
directory = self.directories[0]
# Update log_directory and ingest_cache_directory
if isinstance(config, dict) is True:
if 'main' in config:
# Setup the log_directory to a known good default
(updated, config) = self._create_directory_entries(
'log_directory', config)
updated_list.append(updated)
# Setup the ingest_cache_directory to a known good default
(updated, config) = self._create_directory_entries(
'ingest_cache_directory', config)
updated_list.append(updated)
else:
valid = False
else:
valid = False
# Gracefully exit if things are not OK
if valid is False:
log_message = ('Configuration files found in {} is invalid'
''.format(self.directories))
log.log2die_safe(1101, log_message)
# Update configuration file if required
if len(updated_list) == updated_list.count(True):
for next_directory in self.directories:
# Delete all YAML files in the directory
general.delete_yaml_files(next_directory)
# Write config back to directory
filepath = ('%s/config.yaml') % (directory)
with open(filepath, 'w') as outfile:
yaml.dump(config, outfile, default_flow_style=False)
# Print status
shared.print_ok('Configuration setup complete.')
def run():
"""Setup infoset-ng.
Args:
None
Returns:
None
"""
# Prevent running as sudo user
if 'SUDO_UID' in os.environ:
MESSAGE = ('Cannot run setup using "sudo". Run as a regular user to '
'install in this directory or as user "root".')
log.log2die_safe(1078, MESSAGE)
# Initialize key variables
username = getpass.getuser()
# Determine whether version of python is valid
_PythonSetup().run()
# Do specific setups for root user
_DaemonSetup().run()
# Update configuration if required
_ConfigSetup().run()
# Run server setup
_DatabaseSetup().run()
# Give suggestions as to what to do
if username == 'root':
suggestions = """\
You can start infoset-ng daemons with these commands:
# systemctl start infoset-ng-api.service
# systemctl start infoset-ng-ingester.service
You can enable infoset-ng daemons to start on system boot with these commands:
# systemctl enable infoset-ng-api.service
# systemctl enable infoset-ng-ingester.service
"""
print(suggestions)
# All done
print_ok('Installation successful.')
def _pip3_install(module):
"""Install python module using pip3.
Args:
module: module to install
Returns:
None
"""
# Find pip3 executable
cli_string = 'which pip3'
response = general.run_script(cli_string, die=False)
# Not OK if not fount
if bool(response['returncode']) is True:
log_message = ('python pip3 not installed.')
log.log2die_safe(1041, log_message)
else:
log_message = 'Python pip3 executable found.'
shared.print_ok(log_message)
# Determine version of pip3
cli_string = 'pip3 --version'
_response = os.popen(cli_string).read()
version = _response.split()[1]
# Attempt to install module
if version < '9.0.0':
cli_string = 'pip3 list | grep {}'.format(module)
else:
cli_string = 'pip3 list --format columns | grep {}'.format(module)
__response = bool(os.popen(cli_string).read())
if __response is False:
# YAML is not installed try to install it
cli_string = 'pip3 install --user {}'.format(module)
response_install = general.run_script(cli_string, die=False)
# Fail if module cannot be installed
if bool(response_install['returncode']) is True:
log_message = ('python pip3 cannot install "{}".'.format(module))
log.log2die_safe(1100, log_message)
else:
log_message = ('Python module "{}" is installed.'.format(module))
shared.print_ok(log_message)
else:
log_message = 'Python module "{}" is installed.'.format(module)
shared.print_ok(log_message)
def __init__(self):
"""Function for intializing the class.
Args:
None
Returns:
None
"""
# Initialize key variables
running_username = getpass.getuser()
daemon_username = configuration.Config().username()
self.root_directory = general.root_directory()
infoset_user_exists = True
self.infoset_user = None
self.running_as_root = False
# Set the username we need to be running as
if running_username == 'root':
try:
# Get GID and UID for user
self.infoset_user = daemon_username
self.gid = getpwnam(self.infoset_user).pw_gid
self.uid = getpwnam(self.infoset_user).pw_uid
except KeyError:
infoset_user_exists = False
# Die if user doesn't exist
if infoset_user_exists is False:
log_message = ('User {} not found. Please try again.'
''.format(self.infoset_user))
log.log2die_safe(1049, log_message)
else:
self.infoset_user = daemon_username
# If running as the root user, then the infoset user needs to exist
if running_username == 'root':
self.running_as_root = True
return
def _install_pip3_packages(self):
"""Install PIP3 packages.
Args:
None
Returns:
None
"""
# Initialize key variables
username = self.username
# Don't attempt to install packages if running in the Travis CI
# environment
if 'TRAVIS' in os.environ and 'CI' in os.environ:
return
# Determine whether PIP3 exists
shared.print_ok(
'Installing required pip3 packages from requirements.txt file.')
pip3 = general.search_file('pip3')
if pip3 is None:
log_message = ('Cannot find python "pip3". Please install.')
log.log2die_safe(1052, log_message)
# Install required PIP packages
requirements_file = ('%s/requirements.txt') % (
general.root_directory())
if username == 'root':
script_name = ('pip3 install --upgrade --requirement %s'
'') % (requirements_file)
else:
script_name = ('pip3 install --user --upgrade --requirement %s'
'') % (requirements_file)
general.run_script(script_name)
# Status message
shared.print_ok('pip3 packages installation complete.')
def delete_files(directory, extension='.yaml'):
"""Delete all files of a specfic extension in a directory.
Args:
directory: Directory name
extension: File extension
Returns:
None
"""
# Determine whether directory is valid
if os.path.isdir(directory) is False:
log_message = ('Directory %s does not exist') % (directory)
log.log2die_safe(1023, log_message)
# Get list of files
filelist = [
next_file for next_file in os.listdir(directory)
if next_file.endswith(extension)
]
# Delete files
for delete_file in filelist:
file_path = ('%s/%s') % (directory, delete_file)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
except Exception as exception_error:
log_message = ('Error: deleting files in %s. Error: %s') % (
directory, exception_error)
log.log2die_safe(1014, log_message)
except:
log_message = ('Unexpected error')
log.log2die_safe(1015, log_message)
def read_yaml_files(directories, die=True):
"""Read the contents of all yaml files in a directory.
Args:
directories: List of directory names with configuration files
die: Die if no files found
Returns:
config_dict: Dict of yaml read
"""
# Initialize key variables
yaml_found = False
yaml_from_file = ''
all_yaml_read = ''
# Check each directory in sequence
for config_directory in directories:
# Check if config_directory exists
if os.path.isdir(config_directory) is False:
log_message = (
'Configuration directory "%s" '
'doesn\'t exist!' % config_directory)
log.log2die_safe(1009, log_message)
# Cycle through list of files in directory
for filename in os.listdir(config_directory):
# Examine all the '.yaml' files in directory
if filename.endswith('.yaml'):
# YAML files found
yaml_found = True
# Read file and add to string
file_path = ('%s/%s') % (config_directory, filename)
try:
with open(file_path, 'r') as file_handle:
yaml_from_file = file_handle.read()
except:
log_message = (
'Error reading file %s. Check permissions, '
'existence and file syntax.'
'') % (file_path)
log.log2die_safe(1065, log_message)
# Append yaml from file to all yaml previously read
all_yaml_read = ('%s\n%s') % (all_yaml_read, yaml_from_file)
# Verify YAML files found in directory
if yaml_found is False:
if die is True:
log_message = (
'No files found in directory "%s" with ".yaml" '
'extension.') % (config_directory)
log.log2die_safe(1010, log_message)
# Return
config_dict = list(yaml.safe_load_all(all_yaml_read))[0]
return config_dict
def __init__(self):
"""Function for intializing the class.
Args:
None
Returns:
None
"""
# Initialize key variables
username = getpass.getuser()
self.root_directory = general.root_directory()
self.infoset_user_exists = True
self.infoset_user = None
self.running_as_root = False
# If running as the root user, then the infoset user needs to exist
if username == 'root':
self.running_as_root = True
try:
self.infoset_user = input(
'Please enter the username under which '
'infoset-ng will run: ')
# Get GID and UID for user
self.gid = getpwnam(self.infoset_user).pw_gid
self.uid = getpwnam(self.infoset_user).pw_uid
except KeyError:
self.infoset_user_exists = False
return
# Die if user doesn't exist
if self.infoset_user_exists is False:
log_message = ('User {} not found. Please try again.'
''.format(self.infoset_user))
log.log2die_safe(1049, log_message)
def _file_permissions(self):
"""Set file permissions.
Args:
None
Returns:
None
"""
# Initialize key variables
infoset_user = self.infoset_user
root_directory = self.root_directory
# Prompt to change ownership of root_directory
groupname = grp.getgrgid(self.gid).gr_name
response = input(
'Change ownership of {} directory to user:{} group:{} (y,N) ?: '
''.format(root_directory, infoset_user, groupname))
# Abort if necessary
if response.lower() != 'y':
log_message = ('Aborting as per user request.')
log.log2die_safe(1050, log_message)
# Change ownership of files under root_directory
for parent_directory, directories, files in os.walk(root_directory):
for directory in directories:
os.chown(os.path.join(parent_directory, directory), self.uid,
self.gid)
for next_file in files:
os.chown(os.path.join(parent_directory, next_file), self.uid,
self.gid)
# Change ownership of root_directory
os.chown(root_directory, self.uid, self.gid)
def _key_sub_key(key, sub_key, config_dict, die=True):
"""Get config parameter from YAML.
Args:
key: Primary key
sub_key: Secondary key
config_dict: Dictionary to explore
die: Die if true and the result encountered is None
Returns:
result: result
"""
# Get result
result = None
# Verify config_dict is indeed a dict.
# Die safely as log_directory is not defined
if isinstance(config_dict, dict) is False:
log.log2die_safe(1021, 'Invalid configuration file. YAML not found')
# Get new result
if key in config_dict:
# Make sure we don't have a None value
if config_dict[key] is None:
log_message = (
'Configuration value {}: is blank. Please fix.'
''.format(key))
log.log2die_safe(1129, log_message)
# Get value we need
if sub_key in config_dict[key]:
result = config_dict[key][sub_key]
# Error if not configured
if result is None and die is True:
log_message = (
'%s:%s not defined in configuration') % (key, sub_key)
log.log2die_safe(1016, log_message)
# Return
return result
def __init__(self, daemon_username):
"""Function for intializing the class.
Args:
daemon_username: Username to run scripts as
Returns:
None
"""
# Initialize key variables
valid_directories = []
config = ("""\
main:
log_directory:
log_level: debug
ingest_cache_directory:
ingest_pool_size: 20
listen_address: 0.0.0.0
bind_port: 6000
interval: 300
memcached_hostname: localhost
memcached_port: 11211
sqlalchemy_pool_size: 10
sqlalchemy_max_overflow: 10
db_hostname: localhost
db_username: infoset_ng
db_password:
db_name: infoset_ng
username: {}
""").format(daemon_username)
self.config_dict = yaml.safe_load(config)
directory_dict = defaultdict(lambda: defaultdict(dict))
# Read yaml files from configuration directory
self.directories = general.config_directories()
# Check each directory in sequence
for config_directory in self.directories:
# Check if config_directory exists
if os.path.isdir(config_directory) is False:
continue
# Cycle through list of files in directory
for filename in os.listdir(config_directory):
# Examine all the '.yaml' files in directory
if filename.endswith('.yaml'):
# YAML files found
valid_directories.append(config_directory)
if bool(valid_directories) is True:
directory_dict = general.read_yaml_files(valid_directories,
die=False)
# Populate config_dict with any values found in directory_dict
# Only if the directory has valid files in it.
if bool(directory_dict) is True:
for _main, data_dict in directory_dict.items():
if _main != 'main':
log_message = (
'Invalid files found in configuration directory')
log.log2die_safe(1033, log_message)
for key, value in data_dict.items():
if value is not None:
self.config_dict[_main][key] = value
"""
print(suggestions)
# Outline the versions of MySQL and MariaDB that are required
suggestions = """\
infoset-ng requires:
MySQL >= 5.5
MariaDB >= 10
Please verify.
"""
print(suggestions)
# All done
print('\nOK\n')
if __name__ == '__main__':
# Prevent running as sudo user
if 'SUDO_UID' in os.environ:
MESSAGE = ('Cannot run setup using "sudo". Run as a regular user to '
'install in this directory or as user "root".')
log.log2die_safe(1078, MESSAGE)
# Run main
main()
def _mysql(self):
"""Determine MySQL version.
Args:
None
Returns:
None
"""
# Initialize key variables
valid = True
versions = {
'maria': {
'major': 10,
'minor': 0
},
'mysql': {
'major': 5,
'minor': 7
}
}
# Get response from mysql command
cli_string = '/usr/bin/mysql --version'
response = general.run_script(cli_string)
# Not OK if not fount
if bool(response['returncode']) is True:
valid = False
log_message = ('MySQL / MariaDB not installed.')
log.log2die_safe(1096, log_message)
else:
# Determine major and minor versions of software
version_string = response['stdout'].split()[4]
version_list = version_string.split('.')
major_installed = int(version_list[0])
minor_installed = int(version_list[1])
# We are running MariaDB
if 'maria' in version_string.lower():
major = versions['maria']['major']
minor = versions['maria']['minor']
# Exit if version is too low
if major_installed < major:
valid = False
elif major_installed == major and minor_installed < minor:
valid = False
else:
valid = True
if valid is False:
log_message = ('MariaDB version needs to be >= than {}.{}.'
''.format(major, minor))
log.log2die_safe(1097, log_message)
else:
log_message = ('MariaDB version {}.{}.'
''.format(major_installed, minor_installed))
shared.print_ok(log_message)
# We are running MySQL
else:
major = versions['mysql']['major']
minor = versions['mysql']['minor']
# Exit if version is too low
if major_installed < major:
valid = False
elif major_installed == major and minor_installed < minor:
valid = False
else:
valid = True
if valid is False:
log_message = ('MySQL version needs to be >= than {}.{}.'
''.format(major, minor))
log.log2die_safe(1098, log_message)
else:
log_message = ('MySQL version {}.{}.'
''.format(major_installed, minor_installed))
shared.print_ok(log_message)
# Check whether the server is running
cli_string = 'ps aux | grep /usr/sbin/mysqld | grep -v grep'
response = bool(os.popen(cli_string).read())
# Not OK if not fount
if response is False:
log_message = ('MySQL / MariaDB is not running.')
log.log2die_safe(1099, log_message)
else:
log_message = 'MySQL / MariaDB is running.'
shared.print_ok(log_message)