def test_Serializer():
data = {
'field1': 10,
'field2': 100,
'field3': 1000,
}
s = Serializer()
tmp = tempfile.NamedTemporaryFile('r+',
suffix='.yaml',
dir='/tmp',
delete=False)
try:
s.save_yaml(filename=tmp.name, data=data)
nose.tools.eq_(data, s.load_yaml(filename=tmp.name))
finally:
os.unlink(tmp.name)
tmp = tempfile.NamedTemporaryFile('r+',
suffix='.cpickle',
dir='/tmp',
delete=False)
try:
s.save_cpickle(filename=tmp.name, data=data)
nose.tools.eq_(data, s.load_cpickle(filename=tmp.name))
finally:
os.unlink(tmp.name)
tmp = tempfile.NamedTemporaryFile('r+',
suffix='.json',
dir='/tmp',
delete=False)
try:
s.save_json(filename=tmp.name, data=data)
nose.tools.eq_(data, s.load_json(filename=tmp.name))
finally:
os.unlink(tmp.name)
tmp = tempfile.NamedTemporaryFile('r+',
suffix='.msgpack',
dir='/tmp',
delete=False)
try:
s.save_msgpack(filename=tmp.name, data=data)
nose.tools.eq_(data, s.load_msgpack(filename=tmp.name))
finally:
os.unlink(tmp.name)
tmp = tempfile.NamedTemporaryFile('r+',
suffix='.marshal',
dir='/tmp',
delete=False)
try:
s.save_marshal(filename=tmp.name, data=data)
nose.tools.eq_(data, s.load_marshal(filename=tmp.name))
finally:
os.unlink(tmp.name)
def load(self, filename=None):
"""Load file
Parameters
----------
filename : str, optional
File path
Default value filename given to class constructor
Raises
------
ImportError:
Error if file format specific module cannot be imported
IOError:
File does not exists or has unknown file format
Returns
-------
self
"""
if filename:
self.filename = filename
self.detect_file_format()
self.validate_format()
if self.exists():
# File exits
from dcase_util.files import Serializer
dict.clear(self)
if self.format == FileFormat.YAML or self.format == FileFormat.META:
data = Serializer.load_yaml(filename=self.filename)
dict.update(self, data)
elif self.format == FileFormat.CPICKLE:
dict.update(self,
Serializer.load_cpickle(filename=self.filename))
elif self.format == FileFormat.MARSHAL:
dict.update(self,
Serializer.load_marshal(filename=self.filename))
elif self.format == FileFormat.MSGPACK:
dict.update(self,
Serializer.load_msgpack(filename=self.filename))
elif self.format == FileFormat.JSON:
dict.update(self, Serializer.load_json(filename=self.filename))
elif self.format == FileFormat.TXT:
with open(self.filename, 'r') as f:
lines = f.readlines()
dict.update(self, dict(zip(range(0, len(lines)), lines)))
elif self.format == FileFormat.CSV:
data = {}
delimiter = self.delimiter()
with open(self.filename, 'r') as f:
csv_reader = csv.reader(f, delimiter=delimiter)
for row in csv_reader:
if len(row) == 2:
data[row[0]] = row[1]
dict.update(self, data)
else:
message = '{name}: Unknown format [{format}]'.format(
name=self.__class__.__name__, format=self.filename)
self.logger.exception(message)
raise IOError(message)
else:
message = '{name}: File does not exists [{file}]'.format(
name=self.__class__.__name__, file=self.filename)
self.logger.exception(message)
raise IOError(message)
# Check if after load function is defined, call if found
if hasattr(self, '_after_load'):
self._after_load()
return self
def load(self,
filename=None,
fields=None,
csv_header=True,
file_format=None,
delimiter=None,
convert_numeric_fields=True):
"""Load file
Parameters
----------
filename : str, optional
File path
Default value filename given to class constructor
fields : list of str, optional
List of column names
csv_header : bool, optional
Read field names from first line (header). Used only for CSV formatted files.
Default value True
file_format : FileFormat, optional
Forced file format, use this when there is a miss-match between file extension and file format.
delimiter : str, optional
Forced data delimiter for csv format. If None given, automatic delimiter sniffer used. Use this when sniffer does not work.
convert_numeric_fields : bool, optional
Convert int and float fields to correct type.
Default value True
Raises
------
IOError:
File does not exists or has unknown file format
ValueError:
No fields or csv_header set for CSV formatted file.
Returns
-------
self
"""
if filename:
self.filename = filename
if not file_format:
self.detect_file_format()
self.validate_format()
if file_format and FileFormat.validate_label(label=file_format):
self.format = file_format
if self.exists():
from dcase_util.files import Serializer
if self.format == FileFormat.CSV:
if fields is None and csv_header is None:
message = '{name}: Parameters fields or csv_header has to be set for CSV files.'.format(
name=self.__class__.__name__)
self.logger.exception(message)
raise ValueError(message)
data = []
if not delimiter:
delimiter = self.delimiter()
with open(self.filename, 'r') as f:
csv_reader = csv.reader(f, delimiter=delimiter)
if csv_header:
csv_fields = next(csv_reader)
if fields is None:
fields = csv_fields
for row in csv_reader:
if convert_numeric_fields:
for cell_id, cell_data in enumerate(row):
if is_int(cell_data):
row[cell_id] = int(cell_data)
elif is_float(cell_data):
row[cell_id] = float(cell_data)
data.append(dict(zip(fields, row)))
list.__init__(self, data)
elif self.format == FileFormat.YAML:
data = Serializer.load_yaml(filename=self.filename)
if isinstance(data, list):
list.__init__(self, data)
else:
message = '{name}: YAML data is not in list format.'.format(
name=self.__class__.__name__)
self.logger.exception(message)
raise ImportError(message)
elif self.format == FileFormat.CPICKLE:
list.__init__(self,
Serializer.load_cpickle(filename=self.filename))
else:
message = '{name}: Unknown format [{format}]'.format(
name=self.__class__.__name__, format=self.filename)
self.logger.exception(message)
raise IOError(message)
else:
message = '{name}: File does not exists [{file}]'.format(
name=self.__class__.__name__, file=self.filename)
self.logger.exception(message)
raise IOError(message)
# Check if after load function is defined, call if found
if hasattr(self, '_after_load'):
self._after_load()
return self