def test_normal_setstate(self):
"""Validates that only existing setstate methods are called when
there are no registered state functions in the class chain.
"""
# Validate that unpickling the first class gives us an instance of
# the second class with the appropriate attribute values. It will have
# the default Foo values (because there is no state function to move
# them) and also the default Bar values (since they inherit the
# trait defaults because nothing overwrote the values.)
start = Foo()
test_file = io.BytesIO(pickle.dumps(start, 2))
end = VersionedUnpickler(test_file, updater=TestUpdater()).load()
self.assertIsInstance(end, Bar)
self._assertAttributes(end, 1, (False, 1, 1, "foo"))
self._assertAttributes(end, 2, (True, 2, 2, "bar"))
self._assertAttributes(end, 3, None)
# Validate that unpickling the second class gives us an instance of
# the third class with the appropriate attribute values. It will have
# only the Baz attributes with the Bar values (since the __setstate__
# on Baz converted the Bar attributes to Baz attributes.)
start = Bar()
test_file = io.BytesIO(pickle.dumps(start, 2))
end = VersionedUnpickler(test_file, updater=TestUpdater()).load()
self.assertIsInstance(end, Baz)
self._assertAttributes(end, 2, None)
self._assertAttributes(end, 3, (True, 2, 2, "bar"))
def load_project(pickle_filename, updater_path, application_version, protocol,
max_pass=-1):
""" Reads a project from a pickle file and if necessary will update it to
the latest version of the application.
"""
latest_file = pickle_filename
# Read the pickled project's metadata.
f = open(latest_file, 'rb')
metadata = VersionedUnpickler(f).load(max_pass)
f.close()
project_version = metadata.get('version', False)
if not project_version:
raise ValueError, "Could not read version number from the project file"
logger.debug('Project version: %d, Application version: %d' %
(project_version, application_version))
# here you can temporarily force an upgrade each time for testing ....
# project_version = 0
latest_file = upgrade_project(pickle_filename, updater_path,
project_version, application_version, protocol,
max_pass)
# Finally we can import the project ...
logger.info('loading %s' % latest_file)
i_f = open(latest_file, 'rb')
version = VersionedUnpickler(i_f).load(max_pass)
project = VersionedUnpickler(i_f).load(max_pass)
i_f.close()
return project
def load_project(pickle_filename, updater_path, application_version, protocol,
max_pass=-1):
""" Reads a project from a pickle file and if necessary will update it to
the latest version of the application.
"""
latest_file = pickle_filename
# Read the pickled project's metadata.
f = open(latest_file, 'rb')
metadata = VersionedUnpickler(f).load(max_pass)
f.close()
project_version = metadata.get('version', False)
if not project_version:
raise ValueError("Could not read version number from the project file")
logger.debug('Project version: %d, Application version: %d' %
(project_version, application_version))
# here you can temporarily force an upgrade each time for testing ....
# project_version = 0
latest_file = upgrade_project(pickle_filename, updater_path,
project_version, application_version, protocol,
max_pass)
# Finally we can import the project ...
logger.info('loading %s' % latest_file)
i_f = open(latest_file, 'rb')
version = VersionedUnpickler(i_f).load(max_pass)
project = VersionedUnpickler(i_f).load(max_pass)
i_f.close()
return project
def test_toy_app(self):
a = Application()
a.finder.find()
a.get()
s = pickle.dumps(a)
b = pickle.loads(s)
with self.assertRaisesRegex(
AttributeError,
"'StringFinder' object has no attribute 'data'"):
b.get()
# Works fine.
c = VersionedUnpickler(io.BytesIO(s)).load()
c.get()
def test_unpickled_class_mapping(self):
class TestUpdater(Updater):
def __init__(self):
self.refactorings = {
(Foo.__module__, Foo.__name__):
(Bar.__module__, Bar.__name__),
(Bar.__module__, Bar.__name__):
(Baz.__module__, Baz.__name__),
}
self.setstates = {}
# Validate that unpickling the first class gives us an instance of
# the second class.
start = Foo()
test_file = io.BytesIO(pickle.dumps(start, 2))
end = VersionedUnpickler(test_file, updater=TestUpdater()).load()
self.assertIsInstance(end, Bar)
# Validate that unpickling the second class gives us an instance of
# the third class.
start = Bar()
test_file = io.BytesIO(pickle.dumps(start, 2))
end = VersionedUnpickler(test_file, updater=TestUpdater()).load()
self.assertIsInstance(end, Baz)
def load(self, path):
""" Loads an object from a file. """
# Unpickle the object.
f = open(path, "rb")
try:
try:
obj = VersionedUnpickler(f).load()
except Exception as ex:
print_exc()
logger.exception("Failed to load pickle file: %s, %s" %
(path, ex))
raise
finally:
f.close()
return obj
def test_generic(self):
a = A()
b = B()
a.x = random.randint(1, 100)
b.set_a(a)
a.set_b(b)
value = a.x
# This will fail, even though we have a __setstate__ method.
s = pickle.dumps(a)
new_a = pickle.loads(s)
# Accessing new_a.x is okay
new_a.x
# Accessing y directly would fail
with self.assertRaisesRegex(AttributeError,
"'B' object has no attribute 'y'"):
new_a.b_ref.y
# This will work!
s = pickle.dumps(a)
new_a = VersionedUnpickler(io.BytesIO(s)).load()
assert new_a.x == new_a.b_ref.y == value
def upgrade_project(pickle_filename,
updater_path,
project_version,
application_version,
protocol,
max_pass=-1):
""" Repeatedly read and write the project to disk updating it one version
at a time.
Example the p5.project is at version 0
The application is at version 3
p5.project --- Update1 ---> p5.project.v1
p5.project.v1 --- Update2 ---> p5.project.v2
p5.project.v2 --- Update3 ---> p5.project.v3
p5.project.v3 ---> loaded into app
The user then has the option to save the updated project as p5.project
"""
first_time = True
latest_file = pickle_filename
# update the project until it's version matches the application's
while project_version < application_version:
next_version = project_version + 1
if first_time:
i_f = open(pickle_filename, 'rb')
data = i_f.read()
open('%s.bak' % pickle_filename, 'wb').write(data)
i_f.seek(0) # rewind the file to the start
else:
name = '%s.v%d' % (pickle_filename, project_version)
i_f = open(name, 'rb')
latest_file = name
logger.info('converting %s' % latest_file)
# find this version's updater ...
updater_name = '%s.update%d' % (updater_path, next_version)
__import__(updater_name)
mod = sys.modules[updater_name]
klass = getattr(mod, 'Update%d' % next_version)
updater = klass()
# load and update this version of the project
version = VersionedUnpickler(i_f).load(max_pass)
project = VersionedUnpickler(i_f, updater).load(max_pass)
i_f.close()
# set the project version to be the same as the updater we just
# ran on the unpickled files ...
project.metadata['version'] = next_version
# Persist the updated project ...
name = '%s.v%d' % (pickle_filename, next_version)
latest_file = name
o_f = open(name, 'wb')
pickle.dump(project.metadata, o_f, protocol=protocol)
pickle.dump(project, o_f, protocol=protocol)
o_f.close()
# Bump up the version number of the pickled project...
project_version += 1
first_time = False
return latest_file
def upgrade_project(pickle_filename, updater_path, project_version, application_version, protocol, max_pass=-1):
""" Repeatedly read and write the project to disk updating it one version
at a time.
Example the p5.project is at version 0
The application is at version 3
p5.project --- Update1 ---> p5.project.v1
p5.project.v1 --- Update2 ---> p5.project.v2
p5.project.v2 --- Update3 ---> p5.project.v3
p5.project.v3 ---> loaded into app
The user then has the option to save the updated project as p5.project
"""
first_time = True
latest_file = pickle_filename
# update the project until it's version matches the application's
while project_version < application_version:
next_version = project_version + 1
if first_time:
i_f = open(pickle_filename, 'rb')
data = i_f.read()
open('%s.bak' % pickle_filename, 'wb').write(data)
i_f.seek(0) # rewind the file to the start
else:
name = '%s.v%d' % (pickle_filename, project_version)
i_f = open(name, 'rb')
latest_file = name
logger.info('converting %s' % latest_file)
# find this version's updater ...
updater_name = '%s.update%d' % (updater_path, next_version)
__import__(updater_name)
mod = sys.modules[updater_name]
klass = getattr(mod, 'Update%d' % next_version)
updater = klass()
# load and update this version of the project
version = VersionedUnpickler(i_f).load(max_pass)
project = VersionedUnpickler(i_f, updater).load(max_pass)
i_f.close()
# set the project version to be the same as the updater we just
# ran on the unpickled files ...
project.metadata['version'] = next_version
# Persist the updated project ...
name = '%s.v%d' % (pickle_filename, next_version)
latest_file = name
o_f = open(name, 'wb')
pickle.dump(project.metadata, o_f, protocol=protocol)
pickle.dump(project, o_f, protocol=protocol)
o_f.close()
# Bump up the version number of the pickled project...
project_version += 1
first_time = False
return latest_file
obj = Foo2('duncan child')
t2 = pickle.dumps(obj).replace('Foo2', 'Foo')
save('foo2.txt', t2)
obj = Foo3('duncan child')
t3 = pickle.dumps(obj).replace('Foo3', 'Foo')
save('foo3.txt', t3)
'''
print(
'====================================================================')
from apptools.persistence.versioned_unpickler import VersionedUnpickler
from update1 import Update1
# Try and read them back in ...
f = open('foo0.txt')
import sys
rev = 1
__import__('integrationtests.persistence.update%d' % rev)
mod = sys.modules['integrationtests.persistence.update%d' % rev]
klass = getattr(mod, 'Update%d' % rev)
updater = klass()
print('%s %s' % (rev, updater))
p = VersionedUnpickler(f, updater).load()
print(p)
print('Restored version %s %s' % (p.lastname, p.firstname))
#print(p.set)
