def test_Serializer():
data = {
'field1': 10,
'field2': 100,
'field3': 1000,
}
s = Serializer()
tmp = tempfile.NamedTemporaryFile('r+',
suffix='.yaml',
dir='/tmp',
delete=False)
try:
s.save_yaml(filename=tmp.name, data=data)
nose.tools.eq_(data, s.load_yaml(filename=tmp.name))
finally:
os.unlink(tmp.name)
tmp = tempfile.NamedTemporaryFile('r+',
suffix='.cpickle',
dir='/tmp',
delete=False)
try:
s.save_cpickle(filename=tmp.name, data=data)
nose.tools.eq_(data, s.load_cpickle(filename=tmp.name))
finally:
os.unlink(tmp.name)
tmp = tempfile.NamedTemporaryFile('r+',
suffix='.json',
dir='/tmp',
delete=False)
try:
s.save_json(filename=tmp.name, data=data)
nose.tools.eq_(data, s.load_json(filename=tmp.name))
finally:
os.unlink(tmp.name)
tmp = tempfile.NamedTemporaryFile('r+',
suffix='.msgpack',
dir='/tmp',
delete=False)
try:
s.save_msgpack(filename=tmp.name, data=data)
nose.tools.eq_(data, s.load_msgpack(filename=tmp.name))
finally:
os.unlink(tmp.name)
tmp = tempfile.NamedTemporaryFile('r+',
suffix='.marshal',
dir='/tmp',
delete=False)
try:
s.save_marshal(filename=tmp.name, data=data)
nose.tools.eq_(data, s.load_marshal(filename=tmp.name))
finally:
os.unlink(tmp.name)
def save(self, filename=None):
"""Save file
Parameters
----------
filename : str, optional
File path
Default value filename given to class constructor
Raises
------
ImportError:
Error if file format specific module cannot be imported
IOError:
File has unknown file format
Returns
-------
self
"""
if filename:
self.filename = filename
self.detect_file_format()
self.validate_format()
if self.filename is None or self.filename == '':
message = '{name}: Filename is empty [{filename}]'.format(
name=self.__class__.__name__, filename=self.filename)
self.logger.exception(message)
raise IOError(message)
try:
from dcase_util.files import Serializer
data = dict(self)
if hasattr(self, '__getstate__'):
data.update(dict(self.__getstate__()))
# Check if before save function is defined, call if found
if hasattr(self, '_before_save'):
data = self._before_save(data)
if self.format == FileFormat.YAML:
Serializer.save_yaml(filename=self.filename,
data=self.get_dump_content(data=data))
elif self.format == FileFormat.CPICKLE:
Serializer.save_cpickle(filename=self.filename, data=data)
elif self.format == FileFormat.MARSHAL:
Serializer.save_marshal(filename=self.filename, data=data)
elif self.format == FileFormat.MSGPACK:
Serializer.save_msgpack(filename=self.filename, data=data)
elif self.format == FileFormat.JSON:
Serializer.save_json(filename=self.filename, data=data)
elif self.format == FileFormat.TXT:
with open(self.filename, "w") as text_file:
for line_id in self:
text_file.write(self[line_id])
else:
message = '{name}: Unknown format [{format}]'.format(
name=self.__class__.__name__, format=self.filename)
self.logger.exception(message)
raise IOError(message)
except KeyboardInterrupt:
os.remove(
self.filename
) # Delete the file, since most likely it was not saved fully
raise
# Check if after save function is defined, call if found
if hasattr(self, '_after_save'):
self._after_save()
return self
def save(self,
filename=None,
fields=None,
csv_header=True,
file_format=None,
delimiter=','):
"""Save file
Parameters
----------
filename : str, optional
File path
Default value filename given to class constructor
fields : list of str
Fields in correct order, if none given all field in alphabetical order will be outputted
csv_header : bool
In case of CSV formatted file, first line will contain field names. Names are taken from fields parameter.
Default value True
file_format : FileFormat, optional
Forced file format, use this when there is a miss-match between file extension and file format.
delimiter : str
Delimiter to be used when saving data
Raises
------
IOError:
File has unknown file format
Returns
-------
self
"""
if filename:
self.filename = filename
if not file_format:
self.detect_file_format()
self.validate_format()
if file_format and FileFormat.validate_label(label=file_format):
self.format = file_format
if self.filename is None or self.filename == '':
message = '{name}: Filename is empty [{filename}]'.format(
name=self.__class__.__name__, filename=self.filename)
self.logger.exception(message)
raise IOError(message)
try:
from dcase_util.files import Serializer
if self.format == FileFormat.YAML:
data = copy.deepcopy(list(self))
for item_id, item in enumerate(data):
data[item_id] = self.get_dump_content(data=item)
Serializer.save_yaml(filename=self.filename, data=data)
elif self.format == FileFormat.CSV:
if fields is None:
fields = set()
for item in self:
fields.update(list(item.keys()))
fields = sorted(list(fields))
with open(self.filename, 'w') as csv_file:
csv_writer = csv.writer(csv_file, delimiter=delimiter)
if csv_header:
csv_writer.writerow(fields)
for item in self:
item_values = []
for field in fields:
item_values.append(item[field])
csv_writer.writerow(item_values)
elif self.format == FileFormat.CPICKLE:
Serializer.save_cpickle(filename=self.filename, data=self)
else:
message = '{name}: Unknown format [{format}]'.format(
name=self.__class__.__name__, format=self.filename)
self.logger.exception(message)
raise IOError(message)
except KeyboardInterrupt:
os.remove(
self.filename
) # Delete the file, since most likely it was not saved fully
raise
# Check if after save function is defined, call if found
if hasattr(self, '_after_save'):
self._after_save()
return self
def save(self, data, filename=None):
"""Save file
Parameters
----------
data
Data to be saved
filename : str, optional
File path
Default value filename given to class constructor
Raises
------
ImportError:
Error if file format specific module cannot be imported
IOError:
File has unknown file format
Returns
-------
self
"""
if filename:
self.filename = filename
self.detect_file_format()
self.validate_format()
if self.filename is None or self.filename == '':
message = '{name}: Filename is empty [{filename}]'.format(
name=self.__class__.__name__, filename=self.filename)
self.logger.exception(message)
raise IOError(message)
try:
from dcase_util.files import Serializer
if self.format == FileFormat.YAML:
Serializer.save_yaml(filename=self.filename, data=data)
elif self.format == FileFormat.CPICKLE:
Serializer.save_cpickle(filename=self.filename, data=data)
elif self.format == FileFormat.MARSHAL:
Serializer.save_marshal(filename=self.filename, data=data)
elif self.format == FileFormat.MSGPACK:
Serializer.save_msgpack(filename=self.filename, data=data)
elif self.format == FileFormat.JSON:
Serializer.save_json(filename=self.filename, data=data)
elif self.format == FileFormat.TXT:
with open(self.filename, "w") as text_file:
for line_id in data:
text_file.write(data[line_id])
else:
message = '{name}: Unknown format [{format}]'.format(
name=self.__class__.__name__, format=self.filename)
self.logger.exception(message)
raise IOError(message)
except KeyboardInterrupt:
os.remove(
self.filename
) # Delete the file, since most likely it was not saved fully
raise
return self
