def _load_ini_based_io(path, recursive=False, ini=None,
subini={}, include_core=True,
only_coefficients=False): # pragma: no cover
""" DEPRECATED: For convert a previous version to the new json format
Loads a IOSystem or Extension from a ini files
This function can be used to load a IOSystem or Extension specified in a
ini file. DataFrames (tables) are loaded from text or binary pickle files.
For the latter, the extension .pkl or .pickle is assumed, in all other case
the tables are assumed to be in .txt format.
Parameters
----------
path : string
path or ini file name for the data to load
recursive : boolean, optional
If True, load also the data in the subfolders and add them as
extensions to the IOSystem (in that case path must point to the root).
Only first order subfolders are considered (no subfolders in
subfolders) and if a folder does not contain a ini file it's skipped.
Use the subini parameter in case of multiple ini files in a subfolder.
Attribute name of the extension in the IOSystem are based on the
subfolder name. Default is False
ini : string, optional
If there are several ini files in the root folder, take this one for
loading the data If None (default) take the ini found in the folder,
error if several are found
subini : dict, optional
If there are multiple ini in the subfolder, use the ini given in the
dict. Format: 'subfoldername':'ininame' If a key for a subfolder is
not found or None (default), the ini found in the folder will be taken,
error if several are found
include_core : boolean, optional
If False the load method does not include A, L and Z matrix. This
significantly reduces the required memory if the purpose is only
to analyse the results calculated beforehand.
Returns
-------
IOSystem or Extension class depending on systemtype in the ini file
None in case of errors
"""
# check path and given parameter
ini_file_name = None
path = os.path.abspath(os.path.normpath(path))
if os.path.splitext(path)[1] == '.ini':
(path, ini_file_name) = os.path.split(path)
if ini:
ini_file_name = ini
if not os.path.exists(path):
raise ReadError('Given path does not exist')
return None
if not ini_file_name:
_inifound = False
for file in os.listdir(path):
if os.path.splitext(file)[1] == '.ini':
if _inifound:
raise ReadError(
'Found multiple ini files in folder - specify one')
return None
ini_file_name = file
_inifound = True
# read the ini
io_ini = configparser.RawConfigParser()
io_ini.optionxform = lambda option: option
io_ini.read(os.path.join(path, ini_file_name))
systemtype = io_ini.get('systemtype', 'systemtype', fallback=None)
name = io_ini.get('meta', 'name',
fallback=os.path.splitext(ini_file_name)[0])
if systemtype == 'IOSystem':
ret_system = IOSystem(name=name)
elif systemtype == 'Extension':
ret_system = Extension(name=name)
else:
raise ReadError('System not defined in ini')
return None
for key in io_ini['meta']:
setattr(ret_system, key, io_ini.get('meta', key, fallback=None))
for key in io_ini['files']:
if '_nr_index_col' in key:
continue
if '_nr_header' in key:
continue
if not include_core:
not_to_load = ['A', 'L', 'Z']
if key in not_to_load:
continue
if only_coefficients:
_io = IOSystem()
if key not in _io.__coefficients__ + ['unit']:
continue
file_name = io_ini.get('files', key)
nr_index_col = io_ini.get(
'files', key + '_nr_index_col', fallback=None)
nr_header = io_ini.get('files', key + '_nr_header', fallback=None)
if (nr_index_col is None) or (nr_header is None):
raise ReadError(
'Index or column specification missing for {}'.
format(str(file_name)))
return None
_index_col = list(range(int(nr_index_col)))
_header = list(range(int(nr_header)))
if _index_col == [0]:
_index_col = 0
if _header == [0]:
_header = 0
file = os.path.join(path, file_name)
logging.info('Load data from {}'.format(file))
if (os.path.splitext(file)[1] == '.pkl' or
os.path.splitext(file)[1] == '.pickle'):
setattr(ret_system, key,
pd.read_pickle(file))
else:
setattr(ret_system, key,
pd.read_csv(file,
index_col=_index_col,
header=_header, sep='\t'))
if recursive:
# look for subfolder in the given path
subfolder_list = os.walk(path).__next__()[1]
# loop all subfolder and append extension based on
# ini file in subfolder
for subfolder in subfolder_list:
subini_file_name = subini.get(subfolder)
subpath = os.path.abspath(os.path.join(path, subfolder))
if not subini_file_name:
_inifound = False
for file in os.listdir(subpath):
if os.path.splitext(file)[1] == '.ini':
if _inifound:
raise ReadError(
'Found multiple ini files in subfolder '
'{} - specify one'.format(subpath))
return None
subini_file_name = file
_inifound = True
if not _inifound:
continue
# read the ini
subio_ini = configparser.RawConfigParser()
subio_ini.optionxform = lambda option: option
subio_ini.read(os.path.join(subpath, subini_file_name))
systemtype = subio_ini.get('systemtype', 'systemtype',
fallback=None)
name = subio_ini.get('meta', 'name',
fallback=os.path.splitext(
subini_file_name)[0])
if systemtype == 'IOSystem':
raise ReadError('IOSystem found in subfolder {} - '
'only extensions expected'.format(subpath))
return None
elif systemtype == 'Extension':
sub_system = Extension(name=name)
else:
raise ReadError('System not defined in ini')
return None
for key in subio_ini['meta']:
setattr(sub_system, key, subio_ini.get('meta', key,
fallback=None))
for key in subio_ini['files']:
if '_nr_index_col' in key:
continue
if '_nr_header' in key:
continue
if only_coefficients:
_ext = Extension('temp')
if key not in _ext.__coefficients__ + ['unit']:
continue
file_name = subio_ini.get('files', key)
nr_index_col = subio_ini.get('files', key + '_nr_index_col',
fallback=None)
nr_header = subio_ini.get('files', key + '_nr_header',
fallback=None)
if (nr_index_col is None) or (nr_header is None):
raise ReadError('Index or column specification missing '
'for {}'.format(str(file_name)))
return None
_index_col = list(range(int(nr_index_col)))
_header = list(range(int(nr_header)))
if _index_col == [0]:
_index_col = 0
if _header == [0]:
_header = 0
file = os.path.join(subpath, file_name)
logging.info('Load data from {}'.format(file))
if (os.path.splitext(file)[1] == '.pkl' or
os.path.splitext(file)[1] == '.pickle'):
setattr(sub_system, key,
pd.read_pickle(file))
else:
setattr(sub_system, key,
pd.read_csv(file,
index_col=_index_col,
header=_header,
sep='\t'))
# get valid python name from folder
def clean(varStr):
return re.sub(r'\W|^(?=\d)', '_', str(varStr))
setattr(ret_system, clean(subfolder), sub_system)
return ret_system
def parse_exio_ext(file,
index_col,
name,
drop_compartment=True,
version=None,
year=None,
iosystem=None,
sep=','):
""" Parse an EXIOBASE like extension file into pymrio.Extension
EXIOBASE like extensions files are assumed to have two
rows which are used as columns multiindex (region and sector)
and up to three columns for the row index (see Parameters).
Notes
-----
So far this only parses factor of production extensions F (not
final demand extensions FY nor coeffiecents S).
Parameters
----------
file : string
File to parse
index_col : int
The number of columns (1 to 3) at the beginning of the file
to use as the index. The order of the index_col must be
- 1 index column: ['stressor']
- 2 index columns: ['stressor', 'unit']
- 3 index columns: ['stressor', 'compartment', 'unit']
- > 3: everything up to three index columns will be removed
name : string
Name of the extension
drop_compartment : boolean, optional
If True (default) removes the compartment from the index.
version : string, optional
see pymrio.Extension
iosystem : string, optional
see pymrio.Extension
year : string or int
see pymrio.Extension
sep : string, optional
Delimiter to use; default ','
Returns
-------
pymrio.Extension
with F (and unit if available)
"""
file = os.path.abspath(file)
F = pd.read_table(file,
header=[0, 1],
index_col=list(range(index_col)),
sep=sep)
F.columns.names = ['region', 'sector']
if index_col == 1:
F.index.names = ['stressor']
elif index_col == 2:
F.index.names = ['stressor', 'unit']
elif index_col == 3:
F.index.names = ['stressor', 'compartment', 'unit']
else:
F.reset_index(level=list(range(3, index_col)), drop=True, inplace=True)
F.index.names = ['stressor', 'compartment', 'unit']
unit = None
if index_col > 1:
unit = pd.DataFrame(F.iloc[:, 0].reset_index(level='unit').unit)
F.reset_index(level='unit', drop=True, inplace=True)
if drop_compartment:
F.reset_index(level='compartment', drop=True, inplace=True)
unit.reset_index(level='compartment', drop=True, inplace=True)
return Extension(
name=name,
F=F,
unit=unit,
iosystem=iosystem,
version=version,
year=year,
)
def load(path, include_core=True, path_in_arc=''):
""" Loads a IOSystem or Extension previously saved with pymrio
This function can be used to load a IOSystem or Extension specified in a
metadata file (as defined in DEFAULT_FILE_NAMES['filepara']: metadata.json)
DataFrames (tables) are loaded from text or binary pickle files.
For the latter, the extension .pkl or .pickle is assumed, in all other case
the tables are assumed to be in .txt format.
Parameters
----------
path : pathlib.Path or string
Path or path with para file name for the data to load. This must
either point to the directory containing the uncompressed data or
the location of a compressed zip file with the data. In the
later case the parameter 'path_in_arc' need to be specific to
further indicate the location of the data in the compressed file.
include_core : boolean, optional
If False the load method does not include A, L and Z matrix. This
significantly reduces the required memory if the purpose is only
to analyse the results calculated beforehand.
path_in_arc: string, optional
Path to the data in the zip file (where the fileparameters file is
located). path_in_arc must be given without leading dot and slash;
thus to point to the data in the root of the compressed file pass '',
for data in e.g. the folder 'emissions' pass 'emissions/'. Only used
if parameter 'path' points to an compressed zip file.
Returns
-------
IOSystem or Extension class depending on systemtype in the json file
None in case of errors
"""
path = Path(path)
if not path.exists():
raise ReadError('Given path does not exist')
file_para = get_file_para(path=path, path_in_arc=path_in_arc)
if file_para.content['systemtype'] == GENERIC_NAMES['iosys']:
if zipfile.is_zipfile(str(path)):
ret_system = IOSystem(meta=MRIOMetaData(
location=path,
path_in_arc=os.path.join(file_para.folder,
DEFAULT_FILE_NAMES['metadata'])))
ret_system.meta._add_fileio(
"Loaded IO system from {} - {}".format(path, path_in_arc))
else:
ret_system = IOSystem(meta=MRIOMetaData(
location=path / DEFAULT_FILE_NAMES['metadata']))
ret_system.meta._add_fileio(
"Loaded IO system from {}".format(path))
elif file_para.content['systemtype'] == GENERIC_NAMES['ext']:
ret_system = Extension(file_para.content['name'])
else:
raise ReadError('Type of system no defined in the file parameters')
return None
for key in file_para.content['files']:
if not include_core and key not in ['A', 'L', 'Z']:
continue
file_name = file_para.content['files'][key]['name']
nr_index_col = file_para.content['files'][key]['nr_index_col']
nr_header = file_para.content['files'][key]['nr_header']
_index_col = list(range(int(nr_index_col)))
_header = list(range(int(nr_header)))
_index_col = 0 if _index_col == [0] else _index_col
_header = 0 if _header == [0] else _header
if key == 'FY': # Legacy code to read data saved with version < 0.4
key = 'F_Y'
if zipfile.is_zipfile(str(path)):
full_file_name = os.path.join(file_para.folder, file_name)
logging.info('Load data from {}'.format(full_file_name))
with zipfile.ZipFile(file=str(path)) as zf:
if (os.path.splitext(str(full_file_name))[1] == '.pkl' or
os.path.splitext(str(full_file_name))[1] == '.pickle'):
setattr(ret_system, key,
pd.read_pickle(zf.open(full_file_name)))
else:
setattr(ret_system, key,
pd.read_csv(zf.open(full_file_name),
index_col=_index_col,
header=_header, sep='\t'))
else:
full_file_name = path / file_name
logging.info('Load data from {}'.format(full_file_name))
if (os.path.splitext(str(full_file_name))[1] == '.pkl' or
os.path.splitext(str(full_file_name))[1] == '.pickle'):
setattr(ret_system, key,
pd.read_pickle(full_file_name))
else:
setattr(ret_system, key,
pd.read_csv(full_file_name,
index_col=_index_col,
header=_header,
sep='\t'))
return ret_system
def load(path, include_core=True):
""" Loads a IOSystem or Extension previously saved with pymrio
This function can be used to load a IOSystem or Extension specified in a
ini file. DataFrames (tables) are loaded from text or binary pickle files.
For the latter, the extension .pkl or .pickle is assumed, in all other case
the tables are assumed to be in .txt format.
Parameters
----------
path : string
path or ini file name for the data to load
include_core : boolean, optional
If False the load method does not include A, L and Z matrix. This
significantly reduces the required memory if the purpose is only
to analyse the results calculated beforehand.
Returns
-------
IOSystem or Extension class depending on systemtype in the json file
None in case of errors
"""
path = path.rstrip('\\')
path = os.path.abspath(path)
if not os.path.exists(path):
raise ReadError('Given path does not exist')
return None
para_file_path = os.path.join(path, DEFAULT_FILE_NAMES['filepara'])
if not os.path.isfile(para_file_path):
raise ReadError('No file parameter file found')
return None
with open(para_file_path, 'r') as pf:
file_para = json.load(pf)
if file_para['systemtype'] == GENERIC_NAMES['iosys']:
meta_file_path = os.path.join(path, DEFAULT_FILE_NAMES['metadata'])
ret_system = IOSystem(meta=MRIOMetaData(location=meta_file_path))
ret_system.meta._add_fileio("Loaded IO system from {}".format(path))
elif file_para['systemtype'] == GENERIC_NAMES['ext']:
ret_system = Extension(file_para['name'])
else:
raise ReadError('Type of system no defined in the file parameters')
return None
for key in file_para['files']:
if not include_core:
if key in ['A', 'L', 'Z']:
continue
file_name = file_para['files'][key]['name']
full_file_name = os.path.join(path, file_name)
nr_index_col = file_para['files'][key]['nr_index_col']
nr_header = file_para['files'][key]['nr_header']
logging.info('Load data from {}'.format(full_file_name))
_index_col = list(range(int(nr_index_col)))
_header = list(range(int(nr_header)))
if _index_col == [0]:
_index_col = 0
if _header == [0]:
_header = 0
if (os.path.splitext(full_file_name)[1] == '.pkl'
or os.path.splitext(full_file_name)[1] == '.pickle'):
setattr(ret_system, key, pd.read_pickle(full_file_name))
else:
setattr(
ret_system, key,
pd.read_table(full_file_name,
index_col=_index_col,
header=_header))
return ret_system