def load_data(file_name,
var_names=None,
file_format='infer',
core_name='python_data'):
""" data = load_data( file_name, var_names=None ,
file_format = 'infer' ,
core_name = 'python_data' )
loads dictionary of data from python pickle or matlab struct
Inputs:
file_name - data file name
var_names - variable names to read
file_format - 'infer', 'pickle', or 'matlab'
core_name - data is stored under a dictionary with this name
default looks for variable 'python_data' in file_name
file_format = pickle, will return any python object
file_format = matlab, will return strings or float lists and
requires scipy.io.loadmat
file_format = infer (default), will infer format from extention
('.mat','.pkl')
"""
try:
import scipy.io
scipy_loaded = True
except ImportError:
scipy_loaded = False
if not os.path.exists(file_name):
raise Exception, 'File does not exist: %s' % file_name
# process file format
if file_format == 'infer':
if os.path.splitext(file_name)[1] == '.mat':
file_format = 'matlab'
elif os.path.splitext(file_name)[1] == '.pkl':
file_format = 'pickle'
assert file_format in ['matlab', 'pickle'], 'unsupported file format'
# get filelock
with filelock(file_name):
# LOAD MATLAB
if file_format == 'matlab' and scipy_loaded:
input_data = scipy.io.loadmat(file_name=file_name,
squeeze_me=False,
chars_as_strings=True,
struct_as_record=True)
# pull core variable
assert input_data.has_key(core_name), 'core data not found'
input_data = input_data[core_name]
# convert recarray to dictionary
input_data = rec2dict(input_data)
# LOAD PICKLE
elif file_format == 'pickle':
input_data = load_pickle(file_name)
# pull core variable
assert input_data.has_key(core_name), 'core data not found'
input_data = input_data[core_name]
#: if file_format
#: with filelock
# load specified varname into dictionary
if var_names != None:
# check for one item name array
if isinstance(var_names, str):
var_names = [
var_names,
]
for key in input_data.keys():
if not key in var_names:
del input_data[key]
#: for key
#: if var_names
return input_data
def save_data(file_name,
data_dict,
append=False,
file_format='infer',
core_name='python_data'):
""" save_data( file_name, data_dict, append=False ,
file_format = 'infer' ,
core_name='python_data' ):
Inputs:
file_name - data file name
data_dict - a dictionary or bunch to write
append - True/False to append existing data
file_format - 'infer', 'pickle', or 'matlab'
core_name - data is stored under a dictionary with this name
file_format = pickle, will save any pickleable python object
file_format = matlab, will save strings or float lists and
requires scipy.io.loadmat
file_format = infer (default), will infer format from extention
('.mat','.pkl')
matlab format saves data file from matlab 5 and later
will save nested dictionaries into nested matlab structures
cannot save classes and modules
uses scipy.io.loadmat
"""
try:
import scipy.io
scipy_loaded = True
except ImportError:
scipy_loaded = False
# process file format
if file_format == 'infer':
if os.path.splitext(file_name)[1] == '.mat':
file_format = 'matlab'
elif os.path.splitext(file_name)[1] == '.pkl':
file_format = 'pickle'
assert file_format in ['matlab', 'pickle'], 'unsupported file format'
# get filelock
with filelock(file_name):
# if appending needed
# TODO: don't overwrite other core_names
if append == True and os.path.exists(file_name):
# check file exists
if not os.path.exists(file_name):
raise Exception, 'Cannot append, file does not exist: %s' % file_name
# load old data
data_dict_old = load(file_name=file_name,
var_names=None,
file_format=file_format,
core_name=core_name)
# check for keys not in new data
for key, value in data_dict_old.iteritems():
if not data_dict.has_key(key):
data_dict[key] = value
#: for each dict item
#: if append
# save to core name
data_dict = {core_name: data_dict}
# SAVE MATLAB
if file_format == 'matlab':
# bunch it
data_dict = mat_bunch(data_dict)
# save it
scipy.io.savemat(
file_name=file_name,
mdict=data_dict,
format='5', # matlab 5 .mat format
oned_as='column')
elif file_format == 'pickle':
# save it
save_pickle(file_name, data_dict)
#: if file_format
#: with filelock
return
def load_data( file_name,
file_format = 'infer' ,
core_name = 'python_data' ):
""" data = load_data( file_name,
file_format = 'infer' ,
core_name = 'python_data' )
loads dictionary of data from python pickle or matlab struct
Inputs:
file_name - data file name
file_format - 'infer', 'pickle', or 'matlab'
core_name - data is stored under a dictionary with this name
default looks for variable 'python_data' in file_name
file_format = pickle, will return any python object
file_format = matlab, will return strings or float lists and
requires scipy.io.loadmat
file_format = infer (default), will infer format from extention
('.mat','.pkl')
"""
try:
import scipy.io
scipy_loaded = True
except ImportError:
scipy_loaded = False
if not os.path.exists(file_name):
raise Exception , 'File does not exist: %s' % file_name
# process file format
if file_format == 'infer':
if os.path.splitext(file_name)[1] == '.mat':
file_format = 'matlab'
elif os.path.splitext(file_name)[1] == '.pkl':
file_format = 'pickle'
assert file_format in ['matlab','pickle'] , 'unsupported file format'
# get filelock
with filelock(file_name):
# LOAD MATLAB
if file_format == 'matlab' and scipy_loaded:
input_data = scipy.io.loadmat( file_name = file_name ,
squeeze_me = False ,
chars_as_strings = True ,
struct_as_record = True )
# pull core variable
assert input_data.has_key(core_name) , 'core data field "%s%" not found' % core_name
input_data = input_data[core_name]
# convert recarray to dictionary
input_data = rec2dict(input_data)
# LOAD PICKLE
elif file_format == 'pickle':
input_data = load_pickle(file_name)
# pull core variable
assert input_data.has_key(core_name) , 'core data field "%s%" not found' % core_name
input_data = input_data[core_name]
#: if file_format
#: with filelock
return input_data
def save_data( data_dict, file_name, append=False ,
file_format = 'infer' ,
core_name='python_data' ):
""" save_data( data_dict, file_name, append=False ,
file_format = 'infer' ,
core_name='python_data' ):
Inputs:
file_name - data file name
data_dict - a dictionary or bunch to write
append - True/False to append existing data
file_format - 'infer', 'pickle', or 'matlab'
core_name - data is stored under a dictionary with this name
file_format = pickle, will save any pickleable python object
file_format = matlab, will save strings or float lists and
requires scipy.io.loadmat
file_format = infer (default), will infer format from extention
('.mat','.pkl')
matlab format saves data file from matlab 5 and later
will save nested dictionaries into nested matlab structures
cannot save classes and modules
uses scipy.io.loadmat
"""
try:
import scipy.io
scipy_loaded = True
except ImportError:
scipy_loaded = False
# process file format
if file_format == 'infer':
if os.path.splitext(file_name)[1] == '.mat':
file_format = 'matlab'
elif os.path.splitext(file_name)[1] == '.pkl':
file_format = 'pickle'
assert file_format in ['matlab','pickle'] , 'unsupported file format'
# get filelock
with filelock(file_name):
# if appending needed
# TODO: don't overwrite other core_names
if append == True and os.path.exists(file_name):
# check file exists
if not os.path.exists(file_name):
raise Exception , 'Cannot append, file does not exist: %s' % file_name
# load old data
data_dict_old = load_data( file_name = file_name ,
file_format = file_format ,
core_name = core_name )
# check for keys not in new data
for key,value in data_dict_old.iteritems():
if not data_dict.has_key(key):
data_dict[key] = value
#: for each dict item
#: if append
# save to core name
data_dict = {core_name : data_dict}
# SAVE MATLAB
if file_format == 'matlab':
# bunch it
data_dict = mat_bunch(data_dict)
# save it
scipy.io.savemat( file_name = file_name ,
mdict = data_dict,
format = '5', # matlab 5 .mat format
oned_as = 'column' )
elif file_format == 'pickle':
# save it
save_pickle(file_name,data_dict)
#: if file_format
#: with filelock
return
def load_data(file_name, var_names=None, file_format="infer", core_name="python_data"):
""" data = load_data( file_name, var_names=None ,
file_format = 'infer' ,
core_name = 'python_data' )
loads dictionary of data from python pickle or matlab struct
Inputs:
file_name - data file name
var_names - variable names to read
file_format - 'infer', 'pickle', or 'matlab'
core_name - data is stored under a dictionary with this name
default looks for variable 'python_data' in file_name
file_format = pickle, will return any python object
file_format = matlab, will return strings or float lists and
requires scipy.io.loadmat
file_format = infer (default), will infer format from extention
('.mat','.pkl')
"""
try:
import scipy.io
scipy_loaded = True
except ImportError:
scipy_loaded = False
if not os.path.exists(file_name):
raise Exception, "File does not exist: %s" % file_name
# process file format
if file_format == "infer":
if os.path.splitext(file_name)[1] == ".mat":
file_format = "matlab"
elif os.path.splitext(file_name)[1] == ".pkl":
file_format = "pickle"
assert file_format in ["matlab", "pickle"], "unsupported file format"
# get filelock
with filelock(file_name):
# LOAD MATLAB
if file_format == "matlab" and scipy_loaded:
input_data = scipy.io.loadmat(
file_name=file_name, squeeze_me=False, chars_as_strings=True, struct_as_record=True
)
# pull core variable
assert input_data.has_key(core_name), "core data not found"
input_data = input_data[core_name]
# convert recarray to dictionary
input_data = rec2dict(input_data)
# LOAD PICKLE
elif file_format == "pickle":
input_data = load_pickle(file_name)
# pull core variable
assert input_data.has_key(core_name), "core data not found"
input_data = input_data[core_name]
#: if file_format
#: with filelock
# load specified varname into dictionary
if var_names != None:
# check for one item name array
if isinstance(var_names, str):
var_names = [var_names]
for key in input_data.keys():
if not key in var_names:
del input_data[key]
#: for key
#: if var_names
return input_data