def _check_external_files_from_config(self, add_external_to_object=True):
"""
Read the settings file and check if external config files (i.e.
``calculations_file``, ``html_template``) are defined and exist.
Optionally add all the external files to the Orchestrator object.
"""
conf = arguments.read_config(self.settings_file)
for external_file in ['calculations_file',
'html_template',
'graphs_definition_file']:
# Check if all external files are properly configured
if conf.has_option('MISC', external_file):
value = arguments.get_absolute_path(conf.get('MISC',
external_file),
self.settings_file)
else:
_msg = 'Entry for {0} not found in MISC section (file: {1})'\
.format(external_file, self.settings_file)
raise arguments.ConfigReadError(_msg)
# Check if external files do exist
if not os.path.isfile(value):
_msg = '{0} NOT found: {1}'.format(external_file, value)
raise arguments.ConfigReadError(_msg)
# Update Orchestrator object
if add_external_to_object:
self.__setattr__(external_file, value)
def __init__(self,
alldays=False,
logger=None,
loglevel=None,
nologs=False,
safe=False,
settings_file=None,
**kwargs):
self.alldays = alldays
self.conf = arguments.read_config(settings_file)
self.data = pd.DataFrame()
self.filecache = {}
self.logger = logger or init_logger(loglevel)
self.logs = {}
self.nologs = nologs
self.results_queue = queue.Queue()
self.safe = safe
self.settings_file = settings_file or arguments.DEFAULT_SETTINGS_FILE
self.server = None
self.systems = [
item for item in self.conf.sections()
if item not in ['GATEWAY', 'MISC']
]
add_methods_to_pandas_dataframe(self.logger)
def dump_config(output=None, **kwargs):
"""
Dump current configuration to screen, useful for creating a new
``settings.cfg`` file
Arguments:
output (Optional[str]): output filename, stdout if None
"""
conf = arguments.read_config(**kwargs)
conf.write(output or sys.stdout)
def __init__(self, *args, **kwargs):
super(OrchestratorSandbox, self).__init__(*args, **kwargs)
# Get external files from configuration
self._check_external_files_from_config()
self.folders = []
# Override destination folders to be inside tempdir
conf = read_config(self.settings_file)
for folder in ['reports_folder', 'store_folder']:
# first of all delete the original folders created during __init__
conf_folder = getattr(self, folder)
if conf_folder:
delete_temporary_folder(conf_folder)
value = conf.get('MISC', folder)
setattr(self, folder, path.join(tempfile.gettempdir(), value))
self.folders.append(folder)
def test_config(self):
""" test function for read_config """
config = arguments.read_config(base.TEST_CONFIG)
self.assertIsInstance(config, configparser.ConfigParser)
self.assertGreater(len(config.sections()), 2)
self.assertIn('GATEWAY', config.sections())
self.assertTrue(all([key in [i[0] for i in config.items('DEFAULT')]
for key in ['ssh_port', 'ssh_timeout',
'tunnel_port', 'folder', 'username',
'ip_or_hostname']]))
# Trying to read a bad formatted config file should raise an exception
self.assertRaises(arguments.ConfigReadError,
arguments.read_config,
base.TEST_CSV)
# Same if the file is not found
self.assertRaises(arguments.ConfigReadError,
arguments.read_config,
'badfilename')
def __init__(self, *args, **kwargs):
super(OrchestratorSandbox, self).__init__(*args, **kwargs)
# Get external files from configuration
self._check_external_files_from_config()
self.folders = []
# Override destination folders to be inside tempdir
conf = read_config(self.settings_file)
for folder in ['reports_folder', 'store_folder']:
# first of all delete the original folders created during __init__
conf_folder = getattr(self, folder)
if conf_folder:
delete_temporary_folder(conf_folder)
value = conf.get('MISC', folder)
setattr(self,
folder,
path.join(tempfile.gettempdir(), value))
self.folders.append(folder)
def __init__(self, container, system, logger=None):
self.system = system
# Transparently pass all container items
for item in container.__dict__:
setattr(self, item, getattr(container, item))
if 'loglevel' not in self.__dict__:
self.loglevel = logger.DEFAULT_LOGLEVEL
self.logger = logger or init_logger(self.loglevel)
current_date = dt.datetime.strptime(self.date_time,
"%d/%m/%Y %H:%M:%S")
self.year = current_date.year
# populate self.html_template and self.graphs_definition_file
conf = arguments.read_config(self.settings_file)
for item in ['html_template', 'graphs_definition_file']:
setattr(self,
item,
arguments.get_absolute_path(conf.get('MISC', item),
self.settings_file))
def __init__(self, container, system, logger=None):
self.system = system
# Transparently pass all container items
for item in container.__dict__:
setattr(self, item, getattr(container, item))
if 'loglevel' not in self.__dict__:
self.loglevel = logger.DEFAULT_LOGLEVEL
self.logger = logger or init_logger(self.loglevel)
current_date = dt.datetime.strptime(self.date_time,
"%d/%m/%Y %H:%M:%S")
self.year = current_date.year
# populate self.html_template and self.graphs_definition_file
conf = arguments.read_config(self.settings_file)
for item in ['html_template', 'graphs_definition_file']:
setattr(
self, item,
arguments.get_absolute_path(conf.get('MISC', item),
self.settings_file))
def test_config(self):
""" test function for read_config """
config = arguments.read_config(base.TEST_CONFIG)
self.assertIsInstance(config, configparser.ConfigParser)
self.assertGreater(len(config.sections()), 2)
self.assertIn('GATEWAY', config.sections())
self.assertTrue(
all([
key in [i[0] for i in config.items('DEFAULT')] for key in [
'ssh_port', 'ssh_timeout', 'tunnel_port', 'folder',
'username', 'ip_or_hostname'
]
]))
# Trying to read a bad formatted config file should raise an exception
self.assertRaises(arguments.ConfigReadError, arguments.read_config,
base.TEST_CSV)
# Same if the file is not found
self.assertRaises(arguments.ConfigReadError, arguments.read_config,
'badfilename')
def __init__(self,
alldays=False,
logger=None,
loglevel=None,
nologs=False,
safe=False,
settings_file=None,
**kwargs):
self.alldays = alldays
self.conf = arguments.read_config(settings_file)
self.data = pd.DataFrame()
self.filecache = {}
self.logger = logger or init_logger(loglevel)
self.logs = {}
self.nologs = nologs
self.results_queue = queue.Queue()
self.safe = safe
self.settings_file = settings_file or arguments.DEFAULT_SETTINGS_FILE
self.server = None
self.systems = [item for item in self.conf.sections()
if item not in ['GATEWAY', 'MISC']]
add_methods_to_pandas_dataframe(self.logger)
def _set_folders(self):
"""Setter for reports and store folders and systems"""
self.logger.debug('Using settings file: {0}'
.format(self.settings_file))
conf = arguments.read_config(self.settings_file)
if six.PY3:
self.reports_folder = conf.get('MISC',
'reports_folder',
fallback='./reports')
self.store_folder = conf.get('MISC',
'store_folder',
fallback='./store')
else:
self.reports_folder = conf.get(
'MISC',
'reports_folder'
) if conf.has_option('MISC', 'reports_folder') else './reports'
self.store_folder = conf.get(
'MISC',
'store_folder'
) if conf.has_option('MISC', 'store_folder') else './store'
self.systems = [item for item in conf.sections()
if item not in ['GATEWAY', 'MISC']]
def init_tunnels(self, system=None):
"""
Initialize SSH tunnels using ``sshtunnel`` and ``paramiko`` libraries.
Arguments:
- system
Type: string
Default: ``None``
Description:
system to initialize the tunnels. If nothing given it initializes
tunnels for all systems in ``self.systems``.
Return:
``SSHTunnelForwarder`` instance (non-started) with all tunnels
already established
"""
if not self._check_if_using_gateway(system):
return
self.logger.info('Initializing tunnels')
if not self.conf:
self.conf = arguments.read_config(self.settings_file)
jumpbox_addr = self.conf.get('GATEWAY', 'ip_or_hostname')
jumpbox_port = self.conf.getint('GATEWAY', 'ssh_port')
rbal = []
lbal = []
tunnelports = {}
systems = [system] if system else self.systems
sshtunnel.SSH_TIMEOUT = arguments.DEFAULT_SSH_TIMEOUT
for _sys in systems:
rbal.append((self.conf.get(_sys, 'ip_or_hostname'),
self.conf.getint(_sys, 'ssh_port')))
lbal.append(('', self.conf.getint(_sys, 'tunnel_port')))
if len(tunnelports) != len(set(tunnelports.values())):
self.logger.error('Local tunnel ports MUST be different: {0}'
.format(tunnelports))
raise sshtunnel.BaseSSHTunnelForwarderError
try:
pwd = self.conf.get('GATEWAY', 'password').strip("\"' ") or None \
if self.conf.has_option('GATEWAY', 'password') else None
pkey = self.conf.get('GATEWAY', 'identity_file').strip("\"' ") \
or None if self.conf.has_option('GATEWAY', 'identity_file') \
else None
user = self.conf.get('GATEWAY', 'username') or None \
if self.conf.has_option('GATEWAY', 'username') else None
self.server = sshtunnel.open_tunnel(
ssh_address_or_host=(jumpbox_addr, jumpbox_port),
ssh_username=user,
ssh_password=pwd,
remote_bind_addresses=rbal,
local_bind_addresses=lbal,
threaded=True,
logger=self.logger,
ssh_pkey=pkey,
ssh_private_key_password=pwd,
set_keepalive=15.0,
allow_agent=False,
mute_exceptions=True,
skip_tunnel_checkup=False,
)
self.server.is_use_local_check_up = True # Check local side
self._start_server()
assert self.server.is_alive
# Add the system<>port bindings to the return object
self.server.tunnelports = dict(
list(zip(systems, self.server.local_bind_ports))
)
self.logger.debug('Registered tunnels: {0}'
.format(self.server.tunnelports))
except (sshtunnel.BaseSSHTunnelForwarderError, AssertionError):
self.logger.error('{0}Could not open connection to remote server: '
'{1}:{2}'.format(
'{0} | '.format(system) if system else '',
jumpbox_addr,
jumpbox_port
))
raise sshtunnel.BaseSSHTunnelForwarderError
def init_tunnels(self, system=None):
"""
Initialize SSH tunnels using ``sshtunnel`` and ``paramiko`` libraries.
Arguments:
- system
Type: string
Default: ``None``
Description:
system to initialize the tunnels. If nothing given it initializes
tunnels for all systems in ``self.systems``.
Return:
``SSHTunnelForwarder`` instance (non-started) with all tunnels
already established
"""
if not self._check_if_using_gateway(system):
return
self.logger.info('Initializing tunnels')
if not self.conf:
self.conf = arguments.read_config(self.settings_file)
jumpbox_addr = self.conf.get('GATEWAY', 'ip_or_hostname')
jumpbox_port = self.conf.getint('GATEWAY', 'ssh_port')
rbal = []
lbal = []
tunnelports = {}
systems = [system] if system else self.systems
sshtunnel.SSH_TIMEOUT = arguments.DEFAULT_SSH_TIMEOUT
for _sys in systems:
rbal.append((self.conf.get(_sys, 'ip_or_hostname'),
self.conf.getint(_sys, 'ssh_port')))
lbal.append(('', self.conf.getint(_sys, 'tunnel_port')))
if len(tunnelports) != len(set(tunnelports.values())):
self.logger.error(
'Local tunnel ports MUST be different: {0}'.format(
tunnelports))
raise sshtunnel.BaseSSHTunnelForwarderError
try:
pwd = self.conf.get('GATEWAY', 'password').strip("\"' ") or None \
if self.conf.has_option('GATEWAY', 'password') else None
pkey = self.conf.get('GATEWAY', 'identity_file').strip("\"' ") \
or None if self.conf.has_option('GATEWAY', 'identity_file') \
else None
user = self.conf.get('GATEWAY', 'username') or None \
if self.conf.has_option('GATEWAY', 'username') else None
self.server = sshtunnel.open_tunnel(
ssh_address_or_host=(jumpbox_addr, jumpbox_port),
ssh_username=user,
ssh_password=pwd,
remote_bind_addresses=rbal,
local_bind_addresses=lbal,
threaded=True,
logger=self.logger,
ssh_pkey=pkey,
ssh_private_key_password=pwd,
set_keepalive=15.0,
allow_agent=False,
mute_exceptions=True,
skip_tunnel_checkup=False,
)
self.server.is_use_local_check_up = True # Check local side
self._start_server()
assert self.server.is_alive
# Add the system<>port bindings to the return object
self.server.tunnelports = dict(
list(zip(systems, self.server.local_bind_ports)))
self.logger.debug('Registered tunnels: {0}'.format(
self.server.tunnelports))
except (sshtunnel.BaseSSHTunnelForwarderError, AssertionError):
self.logger.error('{0}Could not open connection to remote server: '
'{1}:{2}'.format(
'{0} | '.format(system) if system else '',
jumpbox_addr, jumpbox_port))
raise sshtunnel.BaseSSHTunnelForwarderError