def configure(args): """ Populate the objects :type args: Namespace :param args: Parsed arguments from argparse :type config_file: str or None :param config_file: Configuration file to read from """ # Add custom configuration file path config_files = [] if args.config: for config in args.config: config = ospath.expanduser(config) if ospath.exists(config): config_files.append(config) LOGGER.info('Added "{}" to config file list'.format(config)) continue LOGGER.warning('Configuration file {} not found.'.format(config)) else: if sys.platform in ['win32', 'cygwin']: config_files.append('C:\\cumulus.conf') config_files.append(ospath.expanduser('~\\.cumulus.conf')) config_files.append('{}\\cumulus.conf'.format(os.curdir)) else: config_files.append('/etc/cumulus.conf') config_files.append(ospath.expanduser('~/.cumulus.conf')) config_files.append('{}/cumulus.conf'.format(os.curdir)) # Get the include option from the general section config_files = __get_include_files(config_files) + config_files # Read config file conf_file_found = False for conf_file in config_files: if ospath.exists(conf_file): conf_file_found = True LOGGER.info('Reading configuration from {}'.format(conf_file)) if not conf_file_found: raise ConfigurationException( 'No configuration file found. Looked for {}'.format( ', '.join(config_files))) config = SafeConfigParser() config.read(config_files) try: _populate_general(args, config) _populate_environments(args, config) _populate_stacks(args, config) _populate_bundles(args, config) except ConfigurationException: raise return CONF
def RemoteCopy(self, local_path, remote_path='', copy_to=True): """Copies a file to or from the VM. Args: local_path: Local path to file. remote_path: Optional path of where to copy file on remote host. copy_to: True to copy to vm, False to copy from vm. Raises: RemoteCommandError: If there was a problem copying the file. """ remote_path = remote_path or '~/' # In order to expand "~" and "~user" we use ntpath.expanduser(), # but it relies on environment variables being set. This modifies # the HOME environment variable in order to use that function, and then # restores it to its previous value. home = os.environ.get('HOME') try: os.environ['HOME'] = self.home_dir remote_path = ntpath.expanduser(remote_path) finally: if home is None: del os.environ['HOME'] else: os.environ['HOME'] = home drive, remote_path = ntpath.splitdrive(remote_path) remote_drive = (drive or self.system_drive).rstrip(':') network_drive = '\\\\%s\\%s$' % (self.GetConnectionIp(), remote_drive) if vm_util.RunningOnWindows(): self._PsDriveRemoteCopy(local_path, remote_path, copy_to, network_drive) else: self._SmbclientRemoteCopy(local_path, remote_path, copy_to, network_drive)
def RemoteCopy(self, local_path, remote_path='', copy_to=True): """Copies a file to or from the VM. Args: local_path: Local path to file. remote_path: Optional path of where to copy file on remote host. copy_to: True to copy to vm, False to copy from vm. Raises: RemoteCommandError: If there was a problem copying the file. """ remote_path = remote_path or '~/' # In order to expand "~" and "~user" we use ntpath.expanduser(), # but it relies on environment variables being set. This modifies # the HOME environment variable in order to use that function, and then # restores it to its previous value. home = os.environ.get('HOME') try: os.environ['HOME'] = self.home_dir remote_path = ntpath.expanduser(remote_path) finally: if home is None: del os.environ['HOME'] else: os.environ['HOME'] = home drive, remote_path = ntpath.splitdrive(remote_path) remote_drive = (drive or self.system_drive).rstrip(':') network_drive = '\\\\%s\\%s$' % (self.ip_address, remote_drive) if vm_util.RunningOnWindows(): self._PsDriveRemoteCopy(local_path, remote_path, copy_to, network_drive) else: self._SmbclientRemoteCopy(local_path, remote_path, copy_to, network_drive)
def RemoteCopy(self, local_path, remote_path='', copy_to=True): """Copies a file to or from the VM. Args: local_path: Local path to file. remote_path: Optional path of where to copy file on remote host. copy_to: True to copy to vm, False to copy from vm. Raises: RemoteCommandError: If there was a problem copying the file. """ remote_path = remote_path or '~/' home = os.environ['HOME'] try: os.environ['HOME'] = self.home_dir remote_path = ntpath.expanduser(remote_path) finally: os.environ['HOME'] = home drive, remote_path = ntpath.splitdrive(remote_path) remote_drive = (drive or self.system_drive).rstrip(':') network_drive = '\\\\%s\\%s$' % (self.ip_address, remote_drive) if vm_util.RunningOnWindows(): self._PsDriveRemoteCopy(local_path, remote_path, copy_to, network_drive) else: self._SmbclientRemoteCopy(local_path, remote_path, copy_to, network_drive)
def main(): parser = argparse.ArgumentParser( description="Convert a SIRA model file in Excel format to JSON.", add_help=True) parser.add_argument("model_file", type=str, help="Path to file to be converted") args = parser.parse_args() excel_file_path = ntpath.expanduser(args.model_file) check_if_excel_file(excel_file_path, parser) try: parent_folder_name = ntpath.dirname(excel_file_path) file_name_full = ntpath.basename(excel_file_path) file_name = os.path.splitext(ntpath.basename(excel_file_path))[0] print("\nConverting system model from MS Excel to JSON...") print("***") print("File Location : {}".format(parent_folder_name)) print("Source file : {}".format(file_name_full)) json_obj = json.loads(read_excel_to_json(excel_file_path), object_pairs_hook=OrderedDict) new_json_structure_obj = update_json_structure(json_obj) json_file_path = os.path.join(parent_folder_name, file_name + '.json') with open(json_file_path, 'w+') as outfile: json.dump(new_json_structure_obj, outfile, indent=4) print("Conversion done : {}".format(ntpath.basename(json_file_path))) print("***") except xlrd.biffh.XLRDError as err: print("Invalid format:", excel_file_path) print(err)
def _populate_bundles(args, config): """ Populate the bundles config object :type args: Namespace :param args: Parsed arguments from argparse :type config: ConfigParser.read :param config: Config parser config object """ for section in config.sections(): if section.startswith('bundle: '): bundle = section.split(': ')[1] CONF['bundles'][bundle] = {} for option, required in BUNDLE_OPTIONS: try: if option == 'paths': lines = config.get(section, option).strip().split('\n') paths = [] for path in lines: paths.append(ospath.expanduser(path.strip())) CONF['bundles'][bundle]['paths'] = paths elif option == 'path-rewrites': CONF['bundles'][bundle]['path-rewrites'] = [] lines = config.get(section, option).strip().split('\n') for line in lines: try: target, destination = line.split('->', 1) except ValueError: raise ConfigurationException( 'Invalid path-rewrites for ' 'bundle {}'.format(bundle)) # Clean the target and destination from initial / if target[0] == '/': target = target[1:] if destination[0] == '/': destination = destination[1:] CONF['bundles'][bundle]['path-rewrites'].append({ 'target': target.strip(), 'destination': destination.strip() }) else: CONF['bundles'][bundle][option] = config.get( section, option) except NoOptionError: if (option == 'paths' and config.has_option( section, 'pre-built-bundle')): continue if required: raise ConfigurationException( 'Missing required option {}'.format(option))
def _populate_bundles(args, config): """ Populate the bundles config object :type args: Namespace :param args: Parsed arguments from argparse :type config: ConfigParser.read :param config: Config parser config object """ for section in config.sections(): if section.startswith('bundle: '): bundle = section.split(': ')[1] CONF['bundles'][bundle] = {} for option, required in BUNDLE_OPTIONS: try: if option == 'paths': lines = config.get(section, option).strip().split('\n') paths = [] for path in lines: paths.append(ospath.expanduser(path.strip())) CONF['bundles'][bundle]['paths'] = paths elif option == 'path-rewrites': CONF['bundles'][bundle]['path-rewrites'] = [] lines = config.get(section, option).strip().split('\n') for line in lines: try: target, destination = line.split('->', 1) except ValueError: raise ConfigurationException( 'Invalid path-rewrites for ' 'bundle {}'.format(bundle)) # Clean the target and destination from initial / if target[0] == '/': target = target[1:] if destination[0] == '/': destination = destination[1:] CONF['bundles'][bundle]['path-rewrites'].append({ 'target': target.strip(), 'destination': destination.strip() }) else: CONF['bundles'][bundle][option] = config.get( section, option) except NoOptionError: if (option == 'paths' and config.has_option(section, 'pre-built-bundle')): continue if required: raise ConfigurationException( 'Missing required option {}'.format(option))
def _get_json_from_template(template): """ Returns a JSON string given a template file path :type template: str :param template: Template path to use :returns: JSON object """ template_path = ospath.expandvars(ospath.expanduser(template)) LOGGER.debug('Parsing template file {}'.format(template_path)) try: with open(template_path) as file_handle: json_data = json.dumps(json.loads(file_handle.read())) except IOError: raise return json_data
def expanduser(path): """Expand ~ and ~user constructions. If user or $HOME is unknown, do nothing.""" if cosmo.kernel == 'nt' and '\\' in path: return ntpath.expanduser(path) path = os.fspath(path) if isinstance(path, bytes): tilde = b'~' else: tilde = '~' if not path.startswith(tilde): return path sep = _get_sep(path) i = path.find(sep, 1) if i < 0: i = len(path) if i == 1: if 'HOME' not in os.environ: import pwd try: userhome = pwd.getpwuid(os.getuid()).pw_dir except KeyError: # bpo-10496: if the current user identifier doesn't exist in the # password database, return the path unchanged return path else: userhome = os.environ['HOME'] else: import pwd name = path[1:i] if isinstance(name, bytes): name = str(name, 'ASCII') try: pwent = pwd.getpwnam(name) except KeyError: # bpo-10496: if the user name from the path doesn't exist in the # password database, return the path unchanged return path userhome = pwent.pw_dir if isinstance(path, bytes): userhome = os.fsencode(userhome) root = b'/' else: root = '/' userhome = userhome.rstrip(root) return (userhome + path[i:]) or root
def browse_picture(self): self.image = QtGui.QFileDialog.getOpenFileName( None, 'OpenFile', 'c:\\', "Image file(*.png *.jpg)") #self.progressBar.setValue(0) self.image = str(self.image) self.labeLAnomaly.setStyleSheet( "QLabel {background-color:red;color:white;}") self.file_name = ntpath.basename(self.image) self.file_name, ext = os.path.splitext(self.file_name) self.file_path = ntpath.dirname(self.image) self.write_path = ntpath.expanduser('~\\Documents\\Document Analysis') # creating write path if not exists if not os.path.exists(self.write_path): os.makedirs(self.write_path) if self.image: self.imgPreview.setPixmap( QtGui.QPixmap(self.image).scaled(self.imgPreview.width(), self.imgPreview.height()))
def RemoteCopy(self, local_path, remote_path='', copy_to=True): """Copies a file to or from the VM. Args: local_path: Local path to file. remote_path: Optional path of where to copy file on remote host. copy_to: True to copy to vm, False to copy from vm. Raises: RemoteCommandError: If there was a problem copying the file. """ # Join with '' to get a trailing \ remote_path = remote_path or ntpath.join(self.home_dir, '') # In order to expand "~" and "~user" we use ntpath.expanduser(), # but it relies on environment variables being set. This modifies # the USERPROFILE environment variable in order to use that function, and # then restores it to its previous value. home = os.environ.get('USERPROFILE') try: os.environ['USERPROFILE'] = self.home_dir remote_path = ntpath.expanduser(remote_path) # Some Windows path's like C:\Users\ADMINI~1 contain ~, # but they should not start with ~ for dir_part in remote_path.split(ntpath.sep): assert not dir_part.startswith( '~'), f'Failed to expand {remote_path}' finally: if home is None: del os.environ['USERPROFILE'] else: os.environ['USERPROFILE'] = home drive, remote_path = ntpath.splitdrive(remote_path) remote_drive = (drive or self.system_drive).rstrip(':') network_drive = '\\\\%s\\%s$' % (self.GetConnectionIp(), remote_drive) if vm_util.RunningOnWindows(): self._PsDriveRemoteCopy(local_path, remote_path, copy_to, network_drive) else: self._SmbclientRemoteCopy(local_path, remote_path, copy_to, network_drive)
def __get_include_files(config_files): """ Read the 'include' option in the 'general' section This will return a list of include files :type config_files: str :param config_files: List of configuration files to include :returns: list -- List of include config files """ config = SafeConfigParser() config.read(config_files) try: return [ ospath.expanduser(c.strip()) for c in config.get('general', 'include').split(',') ] except NoOptionError: return [] except NoSectionError: return [] return []
def _populate_stacks(args, config): """ Populate the stacks config object :type args: Namespace :param args: Parsed arguments from argparse :type config: ConfigParser.read :param config: Config parser config object """ for section in config.sections(): if section.startswith('stack: '): stack = section.split(': ', 1)[1] # If --stacks has been used, do only add those stacks if args.stacks and stack not in args.stacks: continue stack = '{}-{}'.format(args.environment, stack) # Only add stacks that belong to the current environment if stack not in CONF['environments'][args.environment]['stacks']: continue # Prepend a stack name prefix if 'stack-name-prefix' in CONF['environments'][args.environment]: stack = '{}-{}'.format( CONF['environments'][args.environment]['stack-name-prefix'], stack) # Append a stack name suffix if 'stack-name-suffix' in CONF['environments'][args.environment]: stack = '{}-{}'.format( stack, CONF['environments'][args.environment]['stack-name-suffix']) CONF['stacks'][stack] = {} for option, required in STACK_OPTIONS: try: if option == 'disable-rollback': CONF['stacks'][stack][option] = config.getboolean( section, option) elif option == 'template': CONF['stacks'][stack][option] = ospath.expanduser( config.get(section, option)) elif option == 'parameters': try: raw_parameters = config.get(section, option)\ .split('\n') if not raw_parameters[0]: raw_parameters.pop(0) parameters = [] for parameter in raw_parameters: key, value = parameter.split('=', 1) parameters.append((key.strip(), value.strip())) CONF['stacks'][stack][option] = parameters except ValueError: raise ConfigurationException( 'Error parsing parameters for stack {}'.format( stack)) elif option == 'tags': try: raw_tags = config.get(section, option)\ .split('\n') if not raw_tags[0]: raw_tags.pop(0) tags = {} for tag in raw_tags: key, value = tag.split('=', 1) tags[key.strip()] = value.strip() CONF['stacks'][stack][option] = tags except ValueError: raise ConfigurationException( 'Error parsing tags for stack {}'.format( stack)) elif option == 'timeout-in-minutes': CONF['stacks'][stack][option] = config.getint( section, option) else: CONF['stacks'][stack][option] = config.get( section, option) except NoOptionError: if required: raise ConfigurationException( 'Missing required option {}'.format( option)) if option == 'timeout-in-minutes': CONF['stacks'][stack][option] = 0 # Add command line parameters try: if args.parameters: if not 'parameters' in CONF['stacks'][stack]: CONF['stacks'][stack]['parameters'] = [] for raw_parameter in args.parameters.split(','): stack_name, keyvalue = raw_parameter.split(':') key, value = keyvalue.split('=', 1) if stack_name == stack: CONF['stacks'][stack]['parameters'].append( (key, value)) except ValueError: raise ConfigurationException('Error parsing --parameters')
def update_event(self, inp=-1): self.set_output_val(0, ntpath.expanduser(self.input(0)))
# Index all files from within the root #start script if __name__ == '__main__': allfiles = [] sqlite_data = [] # print(f"Indexing files in {os.getcwd()}") # print(f"Reading file list") # for dirpath, dirnames, filenames in os.walk("."): # for filename in [f for f in filenames]: # filelink = os.path.abspath(os.path.join(dirpath, filename)) # if isfile(filelink) and not islink(filelink): # allfiles.append(filelink) args = sys.argv dirs = [] for d in args: p = ntpath.expanduser(d) if isdir(p): dirs.append(p) if (len(dirs) == 0): dirs.append(".") for d in dirs: print(f"Indexing files in {d}") print(f"Reading file list") for dirpath, dirnames, filenames in os.walk(d): for filename in [f for f in filenames]: filelink = os.path.abspath(os.path.join(dirpath, filename)) if isfile(filelink) and not islink(filelink): allfiles.append(filelink) num_of_files = len(allfiles) # "threads" at a time, multiprocess delegation print(f'Checksumming {num_of_files} files (fast)')
def _populate_stacks(args, config): """ Populate the stacks config object :type args: Namespace :param args: Parsed arguments from argparse :type config: ConfigParser.read :param config: Config parser config object """ for section in config.sections(): if section.startswith('stack: '): stack = section.split(': ', 1)[1] # If --stacks has been used, do only add those stacks if args.stacks and stack not in args.stacks: continue stack = '{}-{}'.format(args.environment, stack) # Only add stacks that belong to the current environment if stack not in CONF['environments'][args.environment]['stacks']: continue # Prepend a stack name prefix if 'stack-name-prefix' in CONF['environments'][args.environment]: stack = '{}-{}'.format( CONF['environments'][args.environment] ['stack-name-prefix'], stack) # Append a stack name suffix if 'stack-name-suffix' in CONF['environments'][args.environment]: stack = '{}-{}'.format( stack, CONF['environments'][args.environment] ['stack-name-suffix']) CONF['stacks'][stack] = {} for option, required in STACK_OPTIONS: try: if option == 'disable-rollback': CONF['stacks'][stack][option] = config.getboolean( section, option) elif option == 'template': CONF['stacks'][stack][option] = ospath.expanduser( config.get(section, option)) elif option == 'parameters': try: raw_parameters = config.get(section, option)\ .split('\n') if not raw_parameters[0]: raw_parameters.pop(0) parameters = [] for parameter in raw_parameters: key, value = parameter.split('=', 1) parameters.append((key.strip(), value.strip())) CONF['stacks'][stack][option] = parameters except ValueError: raise ConfigurationException( 'Error parsing parameters for stack {}'.format( stack)) elif option == 'tags': try: raw_tags = config.get(section, option)\ .split('\n') if not raw_tags[0]: raw_tags.pop(0) tags = {} for tag in raw_tags: key, value = tag.split('=', 1) tags[key.strip()] = value.strip() CONF['stacks'][stack][option] = tags except ValueError: raise ConfigurationException( 'Error parsing tags for stack {}'.format( stack)) elif option == 'timeout-in-minutes': CONF['stacks'][stack][option] = config.getint( section, option) else: CONF['stacks'][stack][option] = config.get( section, option) except NoOptionError: if required: raise ConfigurationException( 'Missing required option {}'.format(option)) if option == 'timeout-in-minutes': CONF['stacks'][stack][option] = 0 # Add command line parameters try: if args.parameters: if not 'parameters' in CONF['stacks'][stack]: CONF['stacks'][stack]['parameters'] = [] for raw_parameter in args.parameters.split(','): stack_name, keyvalue = raw_parameter.split(':') key, value = keyvalue.split('=', 1) if stack_name == stack: CONF['stacks'][stack]['parameters'].append( (key, value)) except ValueError: raise ConfigurationException('Error parsing --parameters')