def _systemd(self):
"""Determine if systemd is installed.
Args:
None
Returns:
None
"""
# Initialize key variables.
directory = '/etc/systemd/system'
# Do nothing if this is not the root user.
username = getpass.getuser()
if username != 'root':
return
# Test
if os.path.isdir(directory) is True:
shared.print_ok('Systemd installed.')
else:
log_message = (
'The systemd package isn\'t installed on this system.')
log.log2die_safe(1048, log_message)
def _memcached(self):
"""Determine pip3 version.
Args:
None
Returns:
None
"""
# Find pip3 executable
cli_string = 'which memcached'
response = general.run_script(cli_string)
# Not OK if not found
if bool(response['returncode']) is True:
log_message = (
'memcached is not installed. infoset-ng runs best with it.')
log.log2see_safe(1076, log_message)
else:
log_message = 'memcached executable found.'
shared.print_ok(log_message)
# Check whether the server is running
cli_string = 'ps aux | grep /usr/bin/memcached | grep -v grep'
ps_response = bool(os.popen(cli_string).read())
# Not OK if not fount
if ps_response is False:
log_message = (
'memcached is not running. infoset-ng runs best with it.')
log.log2see_safe(1077, log_message)
else:
log_message = 'memcached is running.'
shared.print_ok(log_message)
def validate(self):
"""Validate .
Args:
None
Returns:
None
"""
# Initialize key variables
username = getpass.getuser()
suggestions = ''
line = '*' * 80
#######################################################################
# Give suggestions as to what to do
#
# NOTE!
#
# The root user should only use the systemctl commands as the daemons
# could be running as another user and lock and pid files will be owned
# by that user. We don't want the daemons to crash at some other time
# because these files are owned by root with denied delete privileges
#######################################################################
if username != 'root':
suggestions = (
'Infoset-NG will not automatically restart after a reboot. '
'You need to re-install as the "root" user for this to occur.')
print('{}\n{}\n{}'.format(line, suggestions, line))
# All done
shared.print_ok(
'Installation complete, pending changes mentioned above.')
def write(self):
"""Write the config to file.
Args:
None
Returns:
None
"""
# Initialize key variables
directory = self.directories[0]
# Update configuration file if required
for next_directory in self.directories:
# Delete all YAML files in the configuration directory
general.delete_yaml_files(next_directory)
# Write config back to directory
filepath = ('%s/config.yaml') % (directory)
with open(filepath, 'w') as outfile:
yaml.dump(self.config_dict, outfile, default_flow_style=False)
# Write status Update
shared.print_ok('Created configuration file {}.'.format(filepath))
def _python(self):
"""Determine Python version.
Args:
None
Returns:
None
"""
# Initialize key variables
valid = True
major = 3
minor = 5
major_installed = sys.version_info[0]
minor_installed = sys.version_info[1]
# Determine whether python version is too low
if major_installed < major:
valid = False
elif major_installed == major and minor_installed < minor:
valid = False
# Process validity
if valid is False:
log_message = ('Required python version must be >= {}.{}. '
'Python version {}.{} installed'
''.format(major, minor, major_installed,
minor_installed))
log.log2die_safe(1095, log_message)
else:
log_message = ('Python version {}.{}.'
''.format(major_installed, minor_installed))
shared.print_ok(log_message)
def run(self):
"""Setup database.
Args:
None
Returns:
None
"""
# Initialize key variables
use_mysql = True
pool_size = 25
max_overflow = 25
config = self.config
mappings = [Agent, Department, Device, Billcode, DeviceAgent, Datapoint, AgentName]
# Create DB connection pool
if use_mysql is True:
# Add MySQL to the pool
engine = create_engine(
URL, echo=False,
encoding='utf8',
max_overflow=max_overflow,
pool_size=pool_size, pool_recycle=3600)
# Try to create the database
shared.print_ok('Attempting to create database tables')
try:
sql_string = (
'ALTER DATABASE %s CHARACTER SET utf8mb4 '
'COLLATE utf8mb4_general_ci') % (config.db_name())
engine.execute(sql_string)
except:
log_message = (
'Cannot connect to database %s. '
'Verify database server is started. '
'Verify database is created. '
'Verify that the configured database authentication '
'is correct.') % (config.db_name())
log.log2die(1046, log_message)
# Apply schemas
shared.print_ok('Generating Schemas.')
with open('infoset.sql', 'w') as infoset_mysql:
for mapping in mappings:
print(CreateTable(mapping.__table__))
infoset_mysql.write(str(CreateTable(mapping.__table__)))
infoset_mysql.close()
# Insert database entries
self._insert_agent_device()
self._insert_billcode()
self._insert_department()
self._insert_datapoint()
self._insert_config()
def run(self):
"""Setup database.
Args:
None
Returns:
None
"""
# Needed to run install subcommand
# https://stackoverflow.com/questions/1321270/how-to-extend-distutils-with-a-simple-post-install-script/1321345#1321345
BuildCommand.run(self)
self.reserved = '_SYSTEM_RESERVED_'
self.config = configuration.Config()
# Initialize key variables
use_mysql = True
pool_size = 25
max_overflow = 25
config = self.config
# Create DB connection pool
if use_mysql is True:
# Add MySQL to the pool
URL = ('sqlite:///%s') % (config.db_file())
engine = create_engine(URL, echo=True, encoding='utf8')
# Try to create the database
shared.print_ok('Attempting to create database tables')
try:
sql_string = ('ALTER DATABASE %s CHARACTER SET utf8mb4 '
'COLLATE utf8mb4_general_ci') % (
config.db_name())
# engine.execute(sql_string)
except:
log_message = (
'Cannot connect to database %s. '
'Verify database server is started. '
'Verify database is created. '
'Verify that the configured database authentication '
'is correct.') % (config.db_name())
log.log2die(1046, log_message)
# Apply schemas
shared.print_ok('Applying Schemas.')
BASE.metadata.create_all(engine)
# Insert database entries
self._insert_agent_device()
self._insert_billcode()
self._insert_department()
self._insert_datapoint()
self._insert_config()
def run(self):
"""Update the configuration with good defaults.
Args:
None
Returns:
None
"""
# Initialize key variables
valid = True
updated_list = []
config = copy.deepcopy(self.config)
directory = self.directories[0]
# Update log_directory and ingest_cache_directory
if isinstance(config, dict) is True:
if 'main' in config:
# Setup the log_directory to a known good default
(updated, config) = self._create_directory_entries(
'log_directory', config)
updated_list.append(updated)
# Setup the ingest_cache_directory to a known good default
(updated, config) = self._create_directory_entries(
'ingest_cache_directory', config)
updated_list.append(updated)
else:
valid = False
else:
valid = False
# Gracefully exit if things are not OK
if valid is False:
log_message = ('Configuration files found in {} is invalid'
''.format(self.directories))
log.log2die_safe(1101, log_message)
# Update configuration file if required
if len(updated_list) == updated_list.count(True):
for next_directory in self.directories:
# Delete all YAML files in the directory
general.delete_yaml_files(next_directory)
# Write config back to directory
filepath = ('%s/config.yaml') % (directory)
with open(filepath, 'w') as outfile:
yaml.dump(config, outfile, default_flow_style=False)
# Print status
shared.print_ok('Configuration setup complete.')
def _db_connectivity(self):
"""Validate we can connect to the database.
Args:
None
Returns:
None
"""
# Initialize key variables
valid = False
db_hostname = self.config_dict['main']['db_hostname']
db_password = self.config_dict['main']['db_password']
db_username = self.config_dict['main']['db_username']
db_name = self.config_dict['main']['db_name']
# Do test
try:
# Open database connection. Prepare cursor
database = pymysql.connect(host=db_hostname,
user=db_username,
passwd=db_password,
db=db_name)
cursor = database.cursor()
# Do a basic check
cursor.execute('SELECT VERSION()')
results = cursor.fetchone()
# Check result
valid = bool(results)
# disconnect from server
database.close()
except Exception as _:
valid = False
except:
valid = False
# Process validity
if valid is True:
log_message = 'Database connectivity successfully verified.'
shared.print_ok(log_message)
else:
log_message = (
'Cannot connect to the database. Verify your credentials. '
'Database Hostname: "{}", Database Username: "******", '
'Database Name: "{}", Database Password: "******"'
''.format(db_hostname, db_username, db_name))
log.log2die_safe(1067, log_message)
def run(self):
"""Setup database.
Args:
None
Returns:
None
"""
# Initialize key variables
use_mysql = True
pool_size = 25
max_overflow = 25
config = self.config
# Create DB connection pool
if use_mysql is True:
# Add MySQL to the pool
engine = create_engine(URL,
echo=True,
encoding='utf8',
max_overflow=max_overflow,
pool_size=pool_size,
pool_recycle=3600)
# Try to create the database
shared.print_ok('Attempting to create database tables')
try:
sql_string = ('ALTER DATABASE %s CHARACTER SET utf8mb4 '
'COLLATE utf8mb4_general_ci') % (
config.db_name())
engine.execute(sql_string)
except:
log_message = (
'Cannot connect to database %s. '
'Verify database server is started. '
'Verify database is created. '
'Verify that the configured database authentication '
'is correct.') % (config.db_name())
log.log2die(1046, log_message)
# Apply schemas
shared.print_ok('Applying Schemas.')
BASE.metadata.create_all(engine)
# Insert database entries
self._insert_agent_device()
self._insert_billcode()
self._insert_department()
self._insert_datapoint()
self._insert_config()
def run():
"""Setup infoset-ng.
Args:
None
Returns:
None
"""
# Run server setup
_DatabaseSetup().run()
# All done
shared.print_ok('Database installation successful.')
def _install_pip3_packages(self):
"""Install PIP3 packages.
Args:
None
Returns:
None
"""
# Initialize key variables
username = self.username
# Don't attempt to install packages if running in the Travis CI
# environment
if 'TRAVIS' in os.environ and 'CI' in os.environ:
return
# Determine whether PIP3 exists
shared.print_ok(
'Installing required pip3 packages from requirements.txt file.')
pip3 = general.search_file('pip3')
if pip3 is None:
log_message = ('Cannot find python "pip3". Please install.')
log.log2die_safe(1052, log_message)
# Install required PIP packages
requirements_file = ('%s/requirements.txt') % (
general.root_directory())
if username == 'root':
script_name = ('pip3 install --upgrade --requirement %s'
'') % (requirements_file)
else:
script_name = ('pip3 install --user --upgrade --requirement %s'
'') % (requirements_file)
general.run_script(script_name)
# Status message
shared.print_ok('pip3 packages installation complete.')
def enable(self):
"""Enable systemd.
Args:
None
Returns:
None
"""
# Return if not running script as root user
if self.running_as_root is False:
return
# Set file permissions
self._file_permissions()
# Setup systemd
self._systemd()
# Determine whether PIP3 exists
shared.print_ok('Daemon setup complete.')
def start(self):
"""Write the config to file.
Args:
None
Returns:
None
"""
# Get daemon status
daemons = ['infoset-ng-api', 'infoset-ng-ingester']
for daemon in daemons:
# Initialize key variables
success = False
running = self._running(daemon)
# Prompt to restart if already running
if running is True:
restart = input(
'\nINPUT - Daemon {} is running. Restart? [Y/n] '
''.format(daemon))
if bool(restart) is False:
success = self._restart(daemon)
if success is True:
shared.print_ok(
'Successfully restarted daemon {}.'.format(daemon))
elif restart[0].lower() != 'n':
success = self._restart(daemon)
if success is True:
shared.print_ok(
'Successfully restarted daemon {}.'.format(daemon))
else:
shared.print_ok(
'Leaving daemon {} unchanged.'.format(daemon))
success = True
else:
success = self._start(daemon)
if success is True:
shared.print_ok(
'Successfully started daemon {}.'.format(daemon))
# Message if no success
if success is False:
log_message = ('Failed to start daemon {}.'.format(daemon))
log.log2see_safe(1008, log_message)
def _pip3_install(module):
"""Install python module using pip3.
Args:
module: module to install
Returns:
None
"""
# Find pip3 executable
cli_string = 'which pip3'
response = general.run_script(cli_string, die=False)
# Not OK if not fount
if bool(response['returncode']) is True:
log_message = ('python pip3 not installed.')
log.log2die_safe(1041, log_message)
else:
log_message = 'Python pip3 executable found.'
shared.print_ok(log_message)
# Determine version of pip3
cli_string = 'pip3 --version'
_response = os.popen(cli_string).read()
version = _response.split()[1]
# Attempt to install module
if version < '9.0.0':
cli_string = 'pip3 list | grep {}'.format(module)
else:
cli_string = 'pip3 list --format columns | grep {}'.format(module)
__response = bool(os.popen(cli_string).read())
if __response is False:
# YAML is not installed try to install it
cli_string = 'pip3 install --user {}'.format(module)
response_install = general.run_script(cli_string, die=False)
# Fail if module cannot be installed
if bool(response_install['returncode']) is True:
log_message = ('python pip3 cannot install "{}".'.format(module))
log.log2die_safe(1100, log_message)
else:
log_message = ('Python module "{}" is installed.'.format(module))
shared.print_ok(log_message)
else:
log_message = 'Python module "{}" is installed.'.format(module)
shared.print_ok(log_message)
def _blank_configuration(self):
"""Determine if systemd is installed.
Args:
None
Returns:
None
"""
# Verify
found = False
root_directory = general.root_directory()
config_directory = '{}/etc'.format(root_directory)
# Do nothing if running Travis CI
if 'INFOSET_CONFIGDIR' in os.environ:
return
# Find yaml files in config_directory
filenames = [
filename for filename in os.listdir(config_directory)
if os.path.isfile(os.path.join(config_directory, filename))
]
for filename in filenames:
if filename.lower().endswith('.yaml'):
log_message = ('Configuration file found')
shared.print_ok(log_message)
found = True
break
# Create a blank config if none are found
if found is False:
config_yaml = ("""\
main:
log_directory:
log_level:
ingest_cache_directory:
ingest_pool_size:
listen_address:
bind_port:
interval:
memcached_hostname:
memcached_port:
sqlalchemy_pool_size:
sqlalchemy_max_overflow:
db_hostname:
db_username:
db_password:
db_name:
username:
""")
# Write config back to directory
config_dict = yaml.safe_load(config_yaml)
filepath = ('%s/config.yaml') % (config_directory)
with open(filepath, 'w') as outfile:
yaml.dump(config_dict, outfile, default_flow_style=False)
# Message
log_message = ('Created blank starter configuration')
shared.print_ok(log_message)
def _mysql(self):
"""Determine MySQL version.
Args:
None
Returns:
None
"""
# Initialize key variables
valid = True
versions = {
'maria': {
'major': 10,
'minor': 0
},
'mysql': {
'major': 5,
'minor': 7
}
}
# Get response from mysql command
cli_string = '/usr/bin/mysql --version'
response = general.run_script(cli_string)
# Not OK if not fount
if bool(response['returncode']) is True:
valid = False
log_message = ('MySQL / MariaDB not installed.')
log.log2die_safe(1096, log_message)
else:
# Determine major and minor versions of software
version_string = response['stdout'].split()[4]
version_list = version_string.split('.')
major_installed = int(version_list[0])
minor_installed = int(version_list[1])
# We are running MariaDB
if 'maria' in version_string.lower():
major = versions['maria']['major']
minor = versions['maria']['minor']
# Exit if version is too low
if major_installed < major:
valid = False
elif major_installed == major and minor_installed < minor:
valid = False
else:
valid = True
if valid is False:
log_message = ('MariaDB version needs to be >= than {}.{}.'
''.format(major, minor))
log.log2die_safe(1097, log_message)
else:
log_message = ('MariaDB version {}.{}.'
''.format(major_installed, minor_installed))
shared.print_ok(log_message)
# We are running MySQL
else:
major = versions['mysql']['major']
minor = versions['mysql']['minor']
# Exit if version is too low
if major_installed < major:
valid = False
elif major_installed == major and minor_installed < minor:
valid = False
else:
valid = True
if valid is False:
log_message = ('MySQL version needs to be >= than {}.{}.'
''.format(major, minor))
log.log2die_safe(1098, log_message)
else:
log_message = ('MySQL version {}.{}.'
''.format(major_installed, minor_installed))
shared.print_ok(log_message)
# Check whether the server is running
cli_string = 'ps aux | grep /usr/sbin/mysqld | grep -v grep'
response = bool(os.popen(cli_string).read())
# Not OK if not fount
if response is False:
log_message = ('MySQL / MariaDB is not running.')
log.log2die_safe(1099, log_message)
else:
log_message = 'MySQL / MariaDB is running.'
shared.print_ok(log_message)
