def _load_ini_based_io(path, recursive=False, ini=None, subini={}, include_core=True, only_coefficients=False): # pragma: no cover """ DEPRECATED: For convert a previous version to the new json format Loads a IOSystem or Extension from a ini files This function can be used to load a IOSystem or Extension specified in a ini file. DataFrames (tables) are loaded from text or binary pickle files. For the latter, the extension .pkl or .pickle is assumed, in all other case the tables are assumed to be in .txt format. Parameters ---------- path : string path or ini file name for the data to load recursive : boolean, optional If True, load also the data in the subfolders and add them as extensions to the IOSystem (in that case path must point to the root). Only first order subfolders are considered (no subfolders in subfolders) and if a folder does not contain a ini file it's skipped. Use the subini parameter in case of multiple ini files in a subfolder. Attribute name of the extension in the IOSystem are based on the subfolder name. Default is False ini : string, optional If there are several ini files in the root folder, take this one for loading the data If None (default) take the ini found in the folder, error if several are found subini : dict, optional If there are multiple ini in the subfolder, use the ini given in the dict. Format: 'subfoldername':'ininame' If a key for a subfolder is not found or None (default), the ini found in the folder will be taken, error if several are found include_core : boolean, optional If False the load method does not include A, L and Z matrix. This significantly reduces the required memory if the purpose is only to analyse the results calculated beforehand. Returns ------- IOSystem or Extension class depending on systemtype in the ini file None in case of errors """ # check path and given parameter ini_file_name = None path = os.path.abspath(os.path.normpath(path)) if os.path.splitext(path)[1] == '.ini': (path, ini_file_name) = os.path.split(path) if ini: ini_file_name = ini if not os.path.exists(path): raise ReadError('Given path does not exist') return None if not ini_file_name: _inifound = False for file in os.listdir(path): if os.path.splitext(file)[1] == '.ini': if _inifound: raise ReadError( 'Found multiple ini files in folder - specify one') return None ini_file_name = file _inifound = True # read the ini io_ini = configparser.RawConfigParser() io_ini.optionxform = lambda option: option io_ini.read(os.path.join(path, ini_file_name)) systemtype = io_ini.get('systemtype', 'systemtype', fallback=None) name = io_ini.get('meta', 'name', fallback=os.path.splitext(ini_file_name)[0]) if systemtype == 'IOSystem': ret_system = IOSystem(name=name) elif systemtype == 'Extension': ret_system = Extension(name=name) else: raise ReadError('System not defined in ini') return None for key in io_ini['meta']: setattr(ret_system, key, io_ini.get('meta', key, fallback=None)) for key in io_ini['files']: if '_nr_index_col' in key: continue if '_nr_header' in key: continue if not include_core: not_to_load = ['A', 'L', 'Z'] if key in not_to_load: continue if only_coefficients: _io = IOSystem() if key not in _io.__coefficients__ + ['unit']: continue file_name = io_ini.get('files', key) nr_index_col = io_ini.get( 'files', key + '_nr_index_col', fallback=None) nr_header = io_ini.get('files', key + '_nr_header', fallback=None) if (nr_index_col is None) or (nr_header is None): raise ReadError( 'Index or column specification missing for {}'. format(str(file_name))) return None _index_col = list(range(int(nr_index_col))) _header = list(range(int(nr_header))) if _index_col == [0]: _index_col = 0 if _header == [0]: _header = 0 file = os.path.join(path, file_name) logging.info('Load data from {}'.format(file)) if (os.path.splitext(file)[1] == '.pkl' or os.path.splitext(file)[1] == '.pickle'): setattr(ret_system, key, pd.read_pickle(file)) else: setattr(ret_system, key, pd.read_csv(file, index_col=_index_col, header=_header, sep='\t')) if recursive: # look for subfolder in the given path subfolder_list = os.walk(path).__next__()[1] # loop all subfolder and append extension based on # ini file in subfolder for subfolder in subfolder_list: subini_file_name = subini.get(subfolder) subpath = os.path.abspath(os.path.join(path, subfolder)) if not subini_file_name: _inifound = False for file in os.listdir(subpath): if os.path.splitext(file)[1] == '.ini': if _inifound: raise ReadError( 'Found multiple ini files in subfolder ' '{} - specify one'.format(subpath)) return None subini_file_name = file _inifound = True if not _inifound: continue # read the ini subio_ini = configparser.RawConfigParser() subio_ini.optionxform = lambda option: option subio_ini.read(os.path.join(subpath, subini_file_name)) systemtype = subio_ini.get('systemtype', 'systemtype', fallback=None) name = subio_ini.get('meta', 'name', fallback=os.path.splitext( subini_file_name)[0]) if systemtype == 'IOSystem': raise ReadError('IOSystem found in subfolder {} - ' 'only extensions expected'.format(subpath)) return None elif systemtype == 'Extension': sub_system = Extension(name=name) else: raise ReadError('System not defined in ini') return None for key in subio_ini['meta']: setattr(sub_system, key, subio_ini.get('meta', key, fallback=None)) for key in subio_ini['files']: if '_nr_index_col' in key: continue if '_nr_header' in key: continue if only_coefficients: _ext = Extension('temp') if key not in _ext.__coefficients__ + ['unit']: continue file_name = subio_ini.get('files', key) nr_index_col = subio_ini.get('files', key + '_nr_index_col', fallback=None) nr_header = subio_ini.get('files', key + '_nr_header', fallback=None) if (nr_index_col is None) or (nr_header is None): raise ReadError('Index or column specification missing ' 'for {}'.format(str(file_name))) return None _index_col = list(range(int(nr_index_col))) _header = list(range(int(nr_header))) if _index_col == [0]: _index_col = 0 if _header == [0]: _header = 0 file = os.path.join(subpath, file_name) logging.info('Load data from {}'.format(file)) if (os.path.splitext(file)[1] == '.pkl' or os.path.splitext(file)[1] == '.pickle'): setattr(sub_system, key, pd.read_pickle(file)) else: setattr(sub_system, key, pd.read_csv(file, index_col=_index_col, header=_header, sep='\t')) # get valid python name from folder def clean(varStr): return re.sub(r'\W|^(?=\d)', '_', str(varStr)) setattr(ret_system, clean(subfolder), sub_system) return ret_system
def parse_exio_ext(file, index_col, name, drop_compartment=True, version=None, year=None, iosystem=None, sep=','): """ Parse an EXIOBASE like extension file into pymrio.Extension EXIOBASE like extensions files are assumed to have two rows which are used as columns multiindex (region and sector) and up to three columns for the row index (see Parameters). Notes ----- So far this only parses factor of production extensions F (not final demand extensions FY nor coeffiecents S). Parameters ---------- file : string File to parse index_col : int The number of columns (1 to 3) at the beginning of the file to use as the index. The order of the index_col must be - 1 index column: ['stressor'] - 2 index columns: ['stressor', 'unit'] - 3 index columns: ['stressor', 'compartment', 'unit'] - > 3: everything up to three index columns will be removed name : string Name of the extension drop_compartment : boolean, optional If True (default) removes the compartment from the index. version : string, optional see pymrio.Extension iosystem : string, optional see pymrio.Extension year : string or int see pymrio.Extension sep : string, optional Delimiter to use; default ',' Returns ------- pymrio.Extension with F (and unit if available) """ file = os.path.abspath(file) F = pd.read_table(file, header=[0, 1], index_col=list(range(index_col)), sep=sep) F.columns.names = ['region', 'sector'] if index_col == 1: F.index.names = ['stressor'] elif index_col == 2: F.index.names = ['stressor', 'unit'] elif index_col == 3: F.index.names = ['stressor', 'compartment', 'unit'] else: F.reset_index(level=list(range(3, index_col)), drop=True, inplace=True) F.index.names = ['stressor', 'compartment', 'unit'] unit = None if index_col > 1: unit = pd.DataFrame(F.iloc[:, 0].reset_index(level='unit').unit) F.reset_index(level='unit', drop=True, inplace=True) if drop_compartment: F.reset_index(level='compartment', drop=True, inplace=True) unit.reset_index(level='compartment', drop=True, inplace=True) return Extension( name=name, F=F, unit=unit, iosystem=iosystem, version=version, year=year, )
def load(path, include_core=True, path_in_arc=''): """ Loads a IOSystem or Extension previously saved with pymrio This function can be used to load a IOSystem or Extension specified in a metadata file (as defined in DEFAULT_FILE_NAMES['filepara']: metadata.json) DataFrames (tables) are loaded from text or binary pickle files. For the latter, the extension .pkl or .pickle is assumed, in all other case the tables are assumed to be in .txt format. Parameters ---------- path : pathlib.Path or string Path or path with para file name for the data to load. This must either point to the directory containing the uncompressed data or the location of a compressed zip file with the data. In the later case the parameter 'path_in_arc' need to be specific to further indicate the location of the data in the compressed file. include_core : boolean, optional If False the load method does not include A, L and Z matrix. This significantly reduces the required memory if the purpose is only to analyse the results calculated beforehand. path_in_arc: string, optional Path to the data in the zip file (where the fileparameters file is located). path_in_arc must be given without leading dot and slash; thus to point to the data in the root of the compressed file pass '', for data in e.g. the folder 'emissions' pass 'emissions/'. Only used if parameter 'path' points to an compressed zip file. Returns ------- IOSystem or Extension class depending on systemtype in the json file None in case of errors """ path = Path(path) if not path.exists(): raise ReadError('Given path does not exist') file_para = get_file_para(path=path, path_in_arc=path_in_arc) if file_para.content['systemtype'] == GENERIC_NAMES['iosys']: if zipfile.is_zipfile(str(path)): ret_system = IOSystem(meta=MRIOMetaData( location=path, path_in_arc=os.path.join(file_para.folder, DEFAULT_FILE_NAMES['metadata']))) ret_system.meta._add_fileio( "Loaded IO system from {} - {}".format(path, path_in_arc)) else: ret_system = IOSystem(meta=MRIOMetaData( location=path / DEFAULT_FILE_NAMES['metadata'])) ret_system.meta._add_fileio( "Loaded IO system from {}".format(path)) elif file_para.content['systemtype'] == GENERIC_NAMES['ext']: ret_system = Extension(file_para.content['name']) else: raise ReadError('Type of system no defined in the file parameters') return None for key in file_para.content['files']: if not include_core and key not in ['A', 'L', 'Z']: continue file_name = file_para.content['files'][key]['name'] nr_index_col = file_para.content['files'][key]['nr_index_col'] nr_header = file_para.content['files'][key]['nr_header'] _index_col = list(range(int(nr_index_col))) _header = list(range(int(nr_header))) _index_col = 0 if _index_col == [0] else _index_col _header = 0 if _header == [0] else _header if key == 'FY': # Legacy code to read data saved with version < 0.4 key = 'F_Y' if zipfile.is_zipfile(str(path)): full_file_name = os.path.join(file_para.folder, file_name) logging.info('Load data from {}'.format(full_file_name)) with zipfile.ZipFile(file=str(path)) as zf: if (os.path.splitext(str(full_file_name))[1] == '.pkl' or os.path.splitext(str(full_file_name))[1] == '.pickle'): setattr(ret_system, key, pd.read_pickle(zf.open(full_file_name))) else: setattr(ret_system, key, pd.read_csv(zf.open(full_file_name), index_col=_index_col, header=_header, sep='\t')) else: full_file_name = path / file_name logging.info('Load data from {}'.format(full_file_name)) if (os.path.splitext(str(full_file_name))[1] == '.pkl' or os.path.splitext(str(full_file_name))[1] == '.pickle'): setattr(ret_system, key, pd.read_pickle(full_file_name)) else: setattr(ret_system, key, pd.read_csv(full_file_name, index_col=_index_col, header=_header, sep='\t')) return ret_system
def load(path, include_core=True): """ Loads a IOSystem or Extension previously saved with pymrio This function can be used to load a IOSystem or Extension specified in a ini file. DataFrames (tables) are loaded from text or binary pickle files. For the latter, the extension .pkl or .pickle is assumed, in all other case the tables are assumed to be in .txt format. Parameters ---------- path : string path or ini file name for the data to load include_core : boolean, optional If False the load method does not include A, L and Z matrix. This significantly reduces the required memory if the purpose is only to analyse the results calculated beforehand. Returns ------- IOSystem or Extension class depending on systemtype in the json file None in case of errors """ path = path.rstrip('\\') path = os.path.abspath(path) if not os.path.exists(path): raise ReadError('Given path does not exist') return None para_file_path = os.path.join(path, DEFAULT_FILE_NAMES['filepara']) if not os.path.isfile(para_file_path): raise ReadError('No file parameter file found') return None with open(para_file_path, 'r') as pf: file_para = json.load(pf) if file_para['systemtype'] == GENERIC_NAMES['iosys']: meta_file_path = os.path.join(path, DEFAULT_FILE_NAMES['metadata']) ret_system = IOSystem(meta=MRIOMetaData(location=meta_file_path)) ret_system.meta._add_fileio("Loaded IO system from {}".format(path)) elif file_para['systemtype'] == GENERIC_NAMES['ext']: ret_system = Extension(file_para['name']) else: raise ReadError('Type of system no defined in the file parameters') return None for key in file_para['files']: if not include_core: if key in ['A', 'L', 'Z']: continue file_name = file_para['files'][key]['name'] full_file_name = os.path.join(path, file_name) nr_index_col = file_para['files'][key]['nr_index_col'] nr_header = file_para['files'][key]['nr_header'] logging.info('Load data from {}'.format(full_file_name)) _index_col = list(range(int(nr_index_col))) _header = list(range(int(nr_header))) if _index_col == [0]: _index_col = 0 if _header == [0]: _header = 0 if (os.path.splitext(full_file_name)[1] == '.pkl' or os.path.splitext(full_file_name)[1] == '.pickle'): setattr(ret_system, key, pd.read_pickle(full_file_name)) else: setattr( ret_system, key, pd.read_table(full_file_name, index_col=_index_col, header=_header)) return ret_system