def configure(args):
""" Populate the objects
:type args: Namespace
:param args: Parsed arguments from argparse
:type config_file: str or None
:param config_file: Configuration file to read from
"""
# Add custom configuration file path
config_files = []
if args.config:
for config in args.config:
config = ospath.expanduser(config)
if ospath.exists(config):
config_files.append(config)
LOGGER.info('Added "{}" to config file list'.format(config))
continue
LOGGER.warning('Configuration file {} not found.'.format(config))
else:
if sys.platform in ['win32', 'cygwin']:
config_files.append('C:\\cumulus.conf')
config_files.append(ospath.expanduser('~\\.cumulus.conf'))
config_files.append('{}\\cumulus.conf'.format(os.curdir))
else:
config_files.append('/etc/cumulus.conf')
config_files.append(ospath.expanduser('~/.cumulus.conf'))
config_files.append('{}/cumulus.conf'.format(os.curdir))
# Get the include option from the general section
config_files = __get_include_files(config_files) + config_files
# Read config file
conf_file_found = False
for conf_file in config_files:
if ospath.exists(conf_file):
conf_file_found = True
LOGGER.info('Reading configuration from {}'.format(conf_file))
if not conf_file_found:
raise ConfigurationException(
'No configuration file found. Looked for {}'.format(
', '.join(config_files)))
config = SafeConfigParser()
config.read(config_files)
try:
_populate_general(args, config)
_populate_environments(args, config)
_populate_stacks(args, config)
_populate_bundles(args, config)
except ConfigurationException:
raise
return CONF
def configure(args):
""" Populate the objects
:type args: Namespace
:param args: Parsed arguments from argparse
:type config_file: str or None
:param config_file: Configuration file to read from
"""
# Add custom configuration file path
config_files = []
if args.config:
for config in args.config:
config = ospath.expanduser(config)
if ospath.exists(config):
config_files.append(config)
LOGGER.info('Added "{}" to config file list'.format(config))
continue
LOGGER.warning('Configuration file {} not found.'.format(config))
else:
if sys.platform in ['win32', 'cygwin']:
config_files.append('C:\\cumulus.conf')
config_files.append(ospath.expanduser('~\\.cumulus.conf'))
config_files.append('{}\\cumulus.conf'.format(os.curdir))
else:
config_files.append('/etc/cumulus.conf')
config_files.append(ospath.expanduser('~/.cumulus.conf'))
config_files.append('{}/cumulus.conf'.format(os.curdir))
# Get the include option from the general section
config_files = __get_include_files(config_files) + config_files
# Read config file
conf_file_found = False
for conf_file in config_files:
if ospath.exists(conf_file):
conf_file_found = True
LOGGER.info('Reading configuration from {}'.format(conf_file))
if not conf_file_found:
raise ConfigurationException(
'No configuration file found. Looked for {}'.format(
', '.join(config_files)))
config = SafeConfigParser()
config.read(config_files)
try:
_populate_general(args, config)
_populate_environments(args, config)
_populate_stacks(args, config)
_populate_bundles(args, config)
except ConfigurationException:
raise
return CONF
def RemoteCopy(self, local_path, remote_path='', copy_to=True):
"""Copies a file to or from the VM.
Args:
local_path: Local path to file.
remote_path: Optional path of where to copy file on remote host.
copy_to: True to copy to vm, False to copy from vm.
Raises:
RemoteCommandError: If there was a problem copying the file.
"""
remote_path = remote_path or '~/'
# In order to expand "~" and "~user" we use ntpath.expanduser(),
# but it relies on environment variables being set. This modifies
# the HOME environment variable in order to use that function, and then
# restores it to its previous value.
home = os.environ.get('HOME')
try:
os.environ['HOME'] = self.home_dir
remote_path = ntpath.expanduser(remote_path)
finally:
if home is None:
del os.environ['HOME']
else:
os.environ['HOME'] = home
drive, remote_path = ntpath.splitdrive(remote_path)
remote_drive = (drive or self.system_drive).rstrip(':')
network_drive = '\\\\%s\\%s$' % (self.GetConnectionIp(), remote_drive)
if vm_util.RunningOnWindows():
self._PsDriveRemoteCopy(local_path, remote_path, copy_to, network_drive)
else:
self._SmbclientRemoteCopy(local_path, remote_path, copy_to, network_drive)
def RemoteCopy(self, local_path, remote_path='', copy_to=True):
"""Copies a file to or from the VM.
Args:
local_path: Local path to file.
remote_path: Optional path of where to copy file on remote host.
copy_to: True to copy to vm, False to copy from vm.
Raises:
RemoteCommandError: If there was a problem copying the file.
"""
remote_path = remote_path or '~/'
# In order to expand "~" and "~user" we use ntpath.expanduser(),
# but it relies on environment variables being set. This modifies
# the HOME environment variable in order to use that function, and then
# restores it to its previous value.
home = os.environ.get('HOME')
try:
os.environ['HOME'] = self.home_dir
remote_path = ntpath.expanduser(remote_path)
finally:
if home is None:
del os.environ['HOME']
else:
os.environ['HOME'] = home
drive, remote_path = ntpath.splitdrive(remote_path)
remote_drive = (drive or self.system_drive).rstrip(':')
network_drive = '\\\\%s\\%s$' % (self.ip_address, remote_drive)
if vm_util.RunningOnWindows():
self._PsDriveRemoteCopy(local_path, remote_path, copy_to, network_drive)
else:
self._SmbclientRemoteCopy(local_path, remote_path, copy_to, network_drive)
def RemoteCopy(self, local_path, remote_path='', copy_to=True):
"""Copies a file to or from the VM.
Args:
local_path: Local path to file.
remote_path: Optional path of where to copy file on remote host.
copy_to: True to copy to vm, False to copy from vm.
Raises:
RemoteCommandError: If there was a problem copying the file.
"""
remote_path = remote_path or '~/'
home = os.environ['HOME']
try:
os.environ['HOME'] = self.home_dir
remote_path = ntpath.expanduser(remote_path)
finally:
os.environ['HOME'] = home
drive, remote_path = ntpath.splitdrive(remote_path)
remote_drive = (drive or self.system_drive).rstrip(':')
network_drive = '\\\\%s\\%s$' % (self.ip_address, remote_drive)
if vm_util.RunningOnWindows():
self._PsDriveRemoteCopy(local_path, remote_path, copy_to,
network_drive)
else:
self._SmbclientRemoteCopy(local_path, remote_path, copy_to,
network_drive)
def main():
parser = argparse.ArgumentParser(
description="Convert a SIRA model file in Excel format to JSON.",
add_help=True)
parser.add_argument("model_file",
type=str,
help="Path to file to be converted")
args = parser.parse_args()
excel_file_path = ntpath.expanduser(args.model_file)
check_if_excel_file(excel_file_path, parser)
try:
parent_folder_name = ntpath.dirname(excel_file_path)
file_name_full = ntpath.basename(excel_file_path)
file_name = os.path.splitext(ntpath.basename(excel_file_path))[0]
print("\nConverting system model from MS Excel to JSON...")
print("***")
print("File Location : {}".format(parent_folder_name))
print("Source file : {}".format(file_name_full))
json_obj = json.loads(read_excel_to_json(excel_file_path),
object_pairs_hook=OrderedDict)
new_json_structure_obj = update_json_structure(json_obj)
json_file_path = os.path.join(parent_folder_name, file_name + '.json')
with open(json_file_path, 'w+') as outfile:
json.dump(new_json_structure_obj, outfile, indent=4)
print("Conversion done : {}".format(ntpath.basename(json_file_path)))
print("***")
except xlrd.biffh.XLRDError as err:
print("Invalid format:", excel_file_path)
print(err)
def _populate_bundles(args, config):
""" Populate the bundles config object
:type args: Namespace
:param args: Parsed arguments from argparse
:type config: ConfigParser.read
:param config: Config parser config object
"""
for section in config.sections():
if section.startswith('bundle: '):
bundle = section.split(': ')[1]
CONF['bundles'][bundle] = {}
for option, required in BUNDLE_OPTIONS:
try:
if option == 'paths':
lines = config.get(section, option).strip().split('\n')
paths = []
for path in lines:
paths.append(ospath.expanduser(path.strip()))
CONF['bundles'][bundle]['paths'] = paths
elif option == 'path-rewrites':
CONF['bundles'][bundle]['path-rewrites'] = []
lines = config.get(section, option).strip().split('\n')
for line in lines:
try:
target, destination = line.split('->', 1)
except ValueError:
raise ConfigurationException(
'Invalid path-rewrites for '
'bundle {}'.format(bundle))
# Clean the target and destination from initial /
if target[0] == '/':
target = target[1:]
if destination[0] == '/':
destination = destination[1:]
CONF['bundles'][bundle]['path-rewrites'].append({
'target':
target.strip(),
'destination':
destination.strip()
})
else:
CONF['bundles'][bundle][option] = config.get(
section, option)
except NoOptionError:
if (option == 'paths' and config.has_option(
section, 'pre-built-bundle')):
continue
if required:
raise ConfigurationException(
'Missing required option {}'.format(option))
def _populate_bundles(args, config):
""" Populate the bundles config object
:type args: Namespace
:param args: Parsed arguments from argparse
:type config: ConfigParser.read
:param config: Config parser config object
"""
for section in config.sections():
if section.startswith('bundle: '):
bundle = section.split(': ')[1]
CONF['bundles'][bundle] = {}
for option, required in BUNDLE_OPTIONS:
try:
if option == 'paths':
lines = config.get(section, option).strip().split('\n')
paths = []
for path in lines:
paths.append(ospath.expanduser(path.strip()))
CONF['bundles'][bundle]['paths'] = paths
elif option == 'path-rewrites':
CONF['bundles'][bundle]['path-rewrites'] = []
lines = config.get(section, option).strip().split('\n')
for line in lines:
try:
target, destination = line.split('->', 1)
except ValueError:
raise ConfigurationException(
'Invalid path-rewrites for '
'bundle {}'.format(bundle))
# Clean the target and destination from initial /
if target[0] == '/':
target = target[1:]
if destination[0] == '/':
destination = destination[1:]
CONF['bundles'][bundle]['path-rewrites'].append({
'target': target.strip(),
'destination': destination.strip()
})
else:
CONF['bundles'][bundle][option] = config.get(
section, option)
except NoOptionError:
if (option == 'paths' and
config.has_option(section, 'pre-built-bundle')):
continue
if required:
raise ConfigurationException(
'Missing required option {}'.format(option))
def _get_json_from_template(template):
""" Returns a JSON string given a template file path
:type template: str
:param template: Template path to use
:returns: JSON object
"""
template_path = ospath.expandvars(ospath.expanduser(template))
LOGGER.debug('Parsing template file {}'.format(template_path))
try:
with open(template_path) as file_handle:
json_data = json.dumps(json.loads(file_handle.read()))
except IOError:
raise
return json_data
def expanduser(path):
"""Expand ~ and ~user constructions. If user or $HOME is unknown,
do nothing."""
if cosmo.kernel == 'nt' and '\\' in path:
return ntpath.expanduser(path)
path = os.fspath(path)
if isinstance(path, bytes):
tilde = b'~'
else:
tilde = '~'
if not path.startswith(tilde):
return path
sep = _get_sep(path)
i = path.find(sep, 1)
if i < 0:
i = len(path)
if i == 1:
if 'HOME' not in os.environ:
import pwd
try:
userhome = pwd.getpwuid(os.getuid()).pw_dir
except KeyError:
# bpo-10496: if the current user identifier doesn't exist in the
# password database, return the path unchanged
return path
else:
userhome = os.environ['HOME']
else:
import pwd
name = path[1:i]
if isinstance(name, bytes):
name = str(name, 'ASCII')
try:
pwent = pwd.getpwnam(name)
except KeyError:
# bpo-10496: if the user name from the path doesn't exist in the
# password database, return the path unchanged
return path
userhome = pwent.pw_dir
if isinstance(path, bytes):
userhome = os.fsencode(userhome)
root = b'/'
else:
root = '/'
userhome = userhome.rstrip(root)
return (userhome + path[i:]) or root
def browse_picture(self):
self.image = QtGui.QFileDialog.getOpenFileName(
None, 'OpenFile', 'c:\\', "Image file(*.png *.jpg)")
#self.progressBar.setValue(0)
self.image = str(self.image)
self.labeLAnomaly.setStyleSheet(
"QLabel {background-color:red;color:white;}")
self.file_name = ntpath.basename(self.image)
self.file_name, ext = os.path.splitext(self.file_name)
self.file_path = ntpath.dirname(self.image)
self.write_path = ntpath.expanduser('~\\Documents\\Document Analysis')
# creating write path if not exists
if not os.path.exists(self.write_path):
os.makedirs(self.write_path)
if self.image:
self.imgPreview.setPixmap(
QtGui.QPixmap(self.image).scaled(self.imgPreview.width(),
self.imgPreview.height()))
def RemoteCopy(self, local_path, remote_path='', copy_to=True):
"""Copies a file to or from the VM.
Args:
local_path: Local path to file.
remote_path: Optional path of where to copy file on remote host.
copy_to: True to copy to vm, False to copy from vm.
Raises:
RemoteCommandError: If there was a problem copying the file.
"""
# Join with '' to get a trailing \
remote_path = remote_path or ntpath.join(self.home_dir, '')
# In order to expand "~" and "~user" we use ntpath.expanduser(),
# but it relies on environment variables being set. This modifies
# the USERPROFILE environment variable in order to use that function, and
# then restores it to its previous value.
home = os.environ.get('USERPROFILE')
try:
os.environ['USERPROFILE'] = self.home_dir
remote_path = ntpath.expanduser(remote_path)
# Some Windows path's like C:\Users\ADMINI~1 contain ~,
# but they should not start with ~
for dir_part in remote_path.split(ntpath.sep):
assert not dir_part.startswith(
'~'), f'Failed to expand {remote_path}'
finally:
if home is None:
del os.environ['USERPROFILE']
else:
os.environ['USERPROFILE'] = home
drive, remote_path = ntpath.splitdrive(remote_path)
remote_drive = (drive or self.system_drive).rstrip(':')
network_drive = '\\\\%s\\%s$' % (self.GetConnectionIp(), remote_drive)
if vm_util.RunningOnWindows():
self._PsDriveRemoteCopy(local_path, remote_path, copy_to,
network_drive)
else:
self._SmbclientRemoteCopy(local_path, remote_path, copy_to,
network_drive)
def __get_include_files(config_files):
""" Read the 'include' option in the 'general' section
This will return a list of include files
:type config_files: str
:param config_files: List of configuration files to include
:returns: list -- List of include config files
"""
config = SafeConfigParser()
config.read(config_files)
try:
return [
ospath.expanduser(c.strip())
for c in config.get('general', 'include').split(',')
]
except NoOptionError:
return []
except NoSectionError:
return []
return []
def __get_include_files(config_files):
""" Read the 'include' option in the 'general' section
This will return a list of include files
:type config_files: str
:param config_files: List of configuration files to include
:returns: list -- List of include config files
"""
config = SafeConfigParser()
config.read(config_files)
try:
return [
ospath.expanduser(c.strip())
for c in config.get('general', 'include').split(',')
]
except NoOptionError:
return []
except NoSectionError:
return []
return []
def _populate_stacks(args, config):
""" Populate the stacks config object
:type args: Namespace
:param args: Parsed arguments from argparse
:type config: ConfigParser.read
:param config: Config parser config object
"""
for section in config.sections():
if section.startswith('stack: '):
stack = section.split(': ', 1)[1]
# If --stacks has been used, do only add those stacks
if args.stacks and stack not in args.stacks:
continue
stack = '{}-{}'.format(args.environment, stack)
# Only add stacks that belong to the current environment
if stack not in CONF['environments'][args.environment]['stacks']:
continue
# Prepend a stack name prefix
if 'stack-name-prefix' in CONF['environments'][args.environment]:
stack = '{}-{}'.format(
CONF['environments'][args.environment]['stack-name-prefix'],
stack)
# Append a stack name suffix
if 'stack-name-suffix' in CONF['environments'][args.environment]:
stack = '{}-{}'.format(
stack,
CONF['environments'][args.environment]['stack-name-suffix'])
CONF['stacks'][stack] = {}
for option, required in STACK_OPTIONS:
try:
if option == 'disable-rollback':
CONF['stacks'][stack][option] = config.getboolean(
section, option)
elif option == 'template':
CONF['stacks'][stack][option] = ospath.expanduser(
config.get(section, option))
elif option == 'parameters':
try:
raw_parameters = config.get(section, option)\
.split('\n')
if not raw_parameters[0]:
raw_parameters.pop(0)
parameters = []
for parameter in raw_parameters:
key, value = parameter.split('=', 1)
parameters.append((key.strip(), value.strip()))
CONF['stacks'][stack][option] = parameters
except ValueError:
raise ConfigurationException(
'Error parsing parameters for stack {}'.format(
stack))
elif option == 'tags':
try:
raw_tags = config.get(section, option)\
.split('\n')
if not raw_tags[0]:
raw_tags.pop(0)
tags = {}
for tag in raw_tags:
key, value = tag.split('=', 1)
tags[key.strip()] = value.strip()
CONF['stacks'][stack][option] = tags
except ValueError:
raise ConfigurationException(
'Error parsing tags for stack {}'.format(
stack))
elif option == 'timeout-in-minutes':
CONF['stacks'][stack][option] = config.getint(
section, option)
else:
CONF['stacks'][stack][option] = config.get(
section, option)
except NoOptionError:
if required:
raise ConfigurationException(
'Missing required option {}'.format(
option))
if option == 'timeout-in-minutes':
CONF['stacks'][stack][option] = 0
# Add command line parameters
try:
if args.parameters:
if not 'parameters' in CONF['stacks'][stack]:
CONF['stacks'][stack]['parameters'] = []
for raw_parameter in args.parameters.split(','):
stack_name, keyvalue = raw_parameter.split(':')
key, value = keyvalue.split('=', 1)
if stack_name == stack:
CONF['stacks'][stack]['parameters'].append(
(key, value))
except ValueError:
raise ConfigurationException('Error parsing --parameters')
def update_event(self, inp=-1):
self.set_output_val(0, ntpath.expanduser(self.input(0)))
# Index all files from within the root
#start script
if __name__ == '__main__':
allfiles = []
sqlite_data = []
# print(f"Indexing files in {os.getcwd()}")
# print(f"Reading file list")
# for dirpath, dirnames, filenames in os.walk("."):
# for filename in [f for f in filenames]:
# filelink = os.path.abspath(os.path.join(dirpath, filename))
# if isfile(filelink) and not islink(filelink):
# allfiles.append(filelink)
args = sys.argv
dirs = []
for d in args:
p = ntpath.expanduser(d)
if isdir(p):
dirs.append(p)
if (len(dirs) == 0):
dirs.append(".")
for d in dirs:
print(f"Indexing files in {d}")
print(f"Reading file list")
for dirpath, dirnames, filenames in os.walk(d):
for filename in [f for f in filenames]:
filelink = os.path.abspath(os.path.join(dirpath, filename))
if isfile(filelink) and not islink(filelink):
allfiles.append(filelink)
num_of_files = len(allfiles)
# "threads" at a time, multiprocess delegation
print(f'Checksumming {num_of_files} files (fast)')
def _populate_stacks(args, config):
""" Populate the stacks config object
:type args: Namespace
:param args: Parsed arguments from argparse
:type config: ConfigParser.read
:param config: Config parser config object
"""
for section in config.sections():
if section.startswith('stack: '):
stack = section.split(': ', 1)[1]
# If --stacks has been used, do only add those stacks
if args.stacks and stack not in args.stacks:
continue
stack = '{}-{}'.format(args.environment, stack)
# Only add stacks that belong to the current environment
if stack not in CONF['environments'][args.environment]['stacks']:
continue
# Prepend a stack name prefix
if 'stack-name-prefix' in CONF['environments'][args.environment]:
stack = '{}-{}'.format(
CONF['environments'][args.environment]
['stack-name-prefix'], stack)
# Append a stack name suffix
if 'stack-name-suffix' in CONF['environments'][args.environment]:
stack = '{}-{}'.format(
stack, CONF['environments'][args.environment]
['stack-name-suffix'])
CONF['stacks'][stack] = {}
for option, required in STACK_OPTIONS:
try:
if option == 'disable-rollback':
CONF['stacks'][stack][option] = config.getboolean(
section, option)
elif option == 'template':
CONF['stacks'][stack][option] = ospath.expanduser(
config.get(section, option))
elif option == 'parameters':
try:
raw_parameters = config.get(section, option)\
.split('\n')
if not raw_parameters[0]:
raw_parameters.pop(0)
parameters = []
for parameter in raw_parameters:
key, value = parameter.split('=', 1)
parameters.append((key.strip(), value.strip()))
CONF['stacks'][stack][option] = parameters
except ValueError:
raise ConfigurationException(
'Error parsing parameters for stack {}'.format(
stack))
elif option == 'tags':
try:
raw_tags = config.get(section, option)\
.split('\n')
if not raw_tags[0]:
raw_tags.pop(0)
tags = {}
for tag in raw_tags:
key, value = tag.split('=', 1)
tags[key.strip()] = value.strip()
CONF['stacks'][stack][option] = tags
except ValueError:
raise ConfigurationException(
'Error parsing tags for stack {}'.format(
stack))
elif option == 'timeout-in-minutes':
CONF['stacks'][stack][option] = config.getint(
section, option)
else:
CONF['stacks'][stack][option] = config.get(
section, option)
except NoOptionError:
if required:
raise ConfigurationException(
'Missing required option {}'.format(option))
if option == 'timeout-in-minutes':
CONF['stacks'][stack][option] = 0
# Add command line parameters
try:
if args.parameters:
if not 'parameters' in CONF['stacks'][stack]:
CONF['stacks'][stack]['parameters'] = []
for raw_parameter in args.parameters.split(','):
stack_name, keyvalue = raw_parameter.split(':')
key, value = keyvalue.split('=', 1)
if stack_name == stack:
CONF['stacks'][stack]['parameters'].append(
(key, value))
except ValueError:
raise ConfigurationException('Error parsing --parameters')
