def test_ndata_handler_basic_functionality(self):
now = datetime.datetime.now()
current_working_directory = os.getcwd()
data_dir = os.path.join(current_working_directory, "__Test")
Cache.db_make_directory_if_needed(data_dir)
try:
h = NDataHandler.NDataHandler(os.path.join(data_dir, "abc.ndata"))
with contextlib.closing(h):
p = {u"abc": 1, u"def": u"bcd", u"uuid": str(uuid.uuid4())}
# write properties
h.write_properties(p, now)
self.assertEqual(h.read_properties(), p)
self.assertIsNone(h.read_data())
# write data
h.write_data(numpy.zeros((4,4), dtype=numpy.float64), now)
self.assertEqual(h.read_properties(), p)
d = h.read_data()
self.assertEqual(d.shape, (4, 4))
self.assertEqual(d.dtype, numpy.float64)
# rewrite data
h.write_data(numpy.zeros((12,12), dtype=numpy.float32), now)
self.assertEqual(h.read_properties(), p)
d = h.read_data()
self.assertEqual(d.shape, (12, 12))
self.assertEqual(d.dtype, numpy.float32)
# rewrite properties
h.write_properties(p, now)
self.assertEqual(h.read_properties(), p)
d = h.read_data()
self.assertEqual(d.shape, (12, 12))
self.assertEqual(d.dtype, numpy.float32)
finally:
#logging.debug("rmtree %s", data_dir)
shutil.rmtree(data_dir)
def test_ndata_handler_rewrites_reversed_zip_file(self):
now = datetime.datetime.now()
current_working_directory = os.getcwd()
data_dir = os.path.join(current_working_directory, "__Test")
Cache.db_make_directory_if_needed(data_dir)
try:
p = {u"abc": 1, u"def": u"bcd", u"uuid": str(uuid.uuid4())}
d = numpy.zeros((12,12), dtype=numpy.float32)
# write zip file where metadata is first
with open(os.path.join(data_dir, "file.ndata"), "w+b") as fp:
dir_data_list = list()
dt = now
properties = p
data = d
if properties is not None:
json_io = io.StringIO()
json.dump(properties, json_io)
json_str = json_io.getvalue()
def write_json(fp):
json_bytes = bytes(json_str, 'ISO-8859-1')
fp.write(json_bytes)
return binascii.crc32(json_bytes) & 0xFFFFFFFF
offset_json = fp.tell()
json_len, json_crc32 = NDataHandler.write_local_file(fp, b"metadata.json", write_json, dt)
dir_data_list.append((offset_json, b"metadata.json", json_len, json_crc32))
if data is not None:
offset_data = fp.tell()
def write_data(fp):
numpy_start_pos = fp.tell()
numpy.save(fp, data)
numpy_end_pos = fp.tell()
fp.seek(numpy_start_pos)
header_data = fp.read((numpy_end_pos - numpy_start_pos) - data.nbytes) # read the header
data_crc32 = binascii.crc32(data.data, binascii.crc32(header_data)) & 0xFFFFFFFF
fp.seek(numpy_end_pos)
return data_crc32
data_len, crc32 = NDataHandler.write_local_file(fp, b"data.npy", write_data, dt)
dir_data_list.append((offset_data, b"data.npy", data_len, crc32))
dir_offset = fp.tell()
for offset, name_bytes, data_len, crc32 in dir_data_list:
NDataHandler.write_directory_data(fp, offset, name_bytes, data_len, crc32, dt)
dir_size = fp.tell() - dir_offset
NDataHandler.write_end_of_directory(fp, dir_size, dir_offset, len(dir_data_list))
fp.truncate()
# make sure read works
h = NDataHandler.NDataHandler(os.path.join(data_dir, "file.ndata"))
with contextlib.closing(h):
self.assertEqual(h.read_properties(), p)
dd = h.read_data()
self.assertEqual(dd.shape, d.shape)
self.assertEqual(dd.dtype, d.dtype)
# now rewrite
h.write_properties(p, now)
finally:
#logging.debug("rmtree %s", data_dir)
shutil.rmtree(data_dir)
def handle_new():
self.library_name = self.__library_name_field.text
workspace_dir = os.path.join(self.directory, self.library_name)
Cache.db_make_directory_if_needed(workspace_dir)
path = os.path.join(workspace_dir, "Nion Swift Workspace.nslib")
if not os.path.exists(path):
with open(path, "w") as fp:
json.dump({}, fp)
if os.path.exists(path):
app.switch_library(workspace_dir)
return True
return False
def test_ndata_handles_corrupt_data(self):
logging.getLogger().setLevel(logging.DEBUG)
now = datetime.datetime.now()
current_working_directory = os.getcwd()
data_dir = os.path.join(current_working_directory, "__Test")
Cache.db_make_directory_if_needed(data_dir)
try:
zero_path = os.path.join(data_dir, "zeros.ndata")
with open(zero_path, 'wb') as f:
f.write(bytearray(1024))
with self.assertRaises(IOError):
with open(zero_path, "rb") as fp:
NDataHandler.parse_zip(fp)
finally:
#logging.debug("rmtree %s", data_dir)
shutil.rmtree(data_dir)
def test_hdf5_handler_basic_functionality(self):
now = datetime.datetime.now()
current_working_directory = pathlib.Path.cwd()
data_dir = current_working_directory / "__Test"
if data_dir.exists():
shutil.rmtree(data_dir)
Cache.db_make_directory_if_needed(data_dir)
try:
h = HDF5Handler.HDF5Handler(os.path.join(data_dir, "abc.h5"))
with contextlib.closing(h):
p = {u"abc": 1, u"def": u"bcd", u"uuid": str(uuid.uuid4())}
# write properties
h.write_properties(p, now)
self.assertEqual(h.read_properties(), p)
self.assertIsNone(h.read_data())
# write data
h.write_data(numpy.zeros((4, 4), dtype=numpy.float64), now)
self.assertEqual(h.read_properties(), p)
d = h.read_data()
self.assertEqual(d.shape, (4, 4))
self.assertEqual(d.dtype, numpy.float64)
# rewrite data
h.write_data(numpy.zeros((12, 12), dtype=numpy.float32), now)
self.assertEqual(h.read_properties(), p)
d = h.read_data()
self.assertEqual(d.shape, (12, 12))
self.assertEqual(d.dtype, numpy.float32)
# rewrite properties
h.write_properties(p, now)
self.assertEqual(h.read_properties(), p)
d = h.read_data()
self.assertEqual(d.shape, (12, 12))
self.assertEqual(d.dtype, numpy.float32)
# reserve data
h.reserve_data((3, 15), numpy.float32, now)
self.assertEqual(h.read_properties(), p)
d = h.read_data()
self.assertEqual(d.shape, (3, 15))
self.assertEqual(d.dtype, numpy.float32)
self.assertTrue(numpy.allclose(d, 0))
finally:
#logging.debug("rmtree %s", data_dir)
shutil.rmtree(data_dir)
def test_switching_library_closes_document_only_once(self):
current_working_directory = os.getcwd()
workspace1_dir = os.path.join(current_working_directory, "__Test1")
workspace2_dir = os.path.join(current_working_directory, "__Test2")
Cache.db_make_directory_if_needed(workspace1_dir)
Cache.db_make_directory_if_needed(workspace2_dir)
try:
app = Application.Application(TestUI.UserInterface(),
set_global=False)
app.initialize(load_plug_ins=False)
app.start(True, fixed_workspace_dir=workspace1_dir)
app.switch_library(workspace2_dir,
skip_choose=True,
fixed_workspace_dir=workspace2_dir)
app.exit()
app.deinitialize()
finally:
#logging.debug("rmtree %s", workspace_dir)
shutil.rmtree(workspace1_dir)
shutil.rmtree(workspace2_dir)
def test_ndata_handles_discontiguous_data(self):
logging.getLogger().setLevel(logging.DEBUG)
now = datetime.datetime.now()
current_working_directory = os.getcwd()
data_dir = os.path.join(current_working_directory, "__Test")
Cache.db_make_directory_if_needed(data_dir)
try:
h = NDataHandler.NDataHandler(os.path.join(data_dir, "abc.ndata"))
with contextlib.closing(h):
data = numpy.random.randint(0, 10, size=(10, 10))[:,3] # discontiguous data
self.assertFalse(data.flags['C_CONTIGUOUS'])
p = {u"uuid": str(uuid.uuid4())}
# write properties
h.write_properties(p, now)
# write data
h.write_data(data, now)
d = h.read_data()
self.assertEqual(d.shape, data.shape)
self.assertEqual(d.dtype, data.dtype)
finally:
#logging.debug("rmtree %s", data_dir)
shutil.rmtree(data_dir)
def choose_library(self):
pose_open_library_dialog_fn = self.pose_open_library_dialog
workspace_history = self.ui.get_persistent_object("workspace_history", list())
nslib_paths = [os.path.join(file_path, "Nion Swift Workspace.nslib") for file_path in workspace_history]
items = [(path, datetime.datetime.fromtimestamp(os.path.getmtime(path))) for path in nslib_paths if os.path.exists(path)]
class ChooseLibraryDialog(Dialog.ActionDialog):
def __init__(self, ui, app):
super().__init__(ui, _("Choose Library"))
current_item_ref = [None]
def handle_choose():
current_item = current_item_ref[0]
if current_item:
app.switch_library(current_item)
return True
return False
def handle_new():
workspace_dir = pose_open_library_dialog_fn()
if workspace_dir:
items.insert(0, (workspace_dir, datetime.datetime.now()))
list_widget.items = items
list_widget.set_selected_index(0)
app.switch_library(workspace_dir)
return True
return False
self.add_button(_("New..."), handle_new)
self.add_button(_("Other..."), handle_new)
self.add_button(_("Cancel"), lambda: True)
self.add_button(_("Choose"), handle_choose)
path_label = ui.create_label_widget()
prompt_row = ui.create_row_widget()
prompt_row.add_spacing(13)
prompt_row.add(ui.create_label_widget(_("Which library do you want Nion Swift to use?"), properties={"stylesheet": "font-weight: bold"}))
prompt_row.add_spacing(13)
prompt_row.add_stretch()
explanation1_row = ui.create_row_widget()
explanation1_row.add_spacing(13)
explanation1_row.add(ui.create_label_widget(_("You can select a library from the list, find another library, or create a new library.")))
explanation1_row.add_spacing(13)
explanation1_row.add_stretch()
explanation2_row = ui.create_row_widget()
explanation2_row.add_spacing(13)
explanation2_row.add(ui.create_label_widget(_("The same library will be used the next time you open Nion Swift.")))
explanation2_row.add_spacing(13)
explanation2_row.add_stretch()
def selection_changed(indexes):
if len(indexes) == 1:
item = items[list(indexes)[0]]
current_item_ref[0] = os.path.dirname(item[0])
path_label.text = os.path.dirname(item[0])
else:
current_item_ref[0] = None
path_label.text = None
def stringify_item(item):
date_utc = item[1]
tz_minutes = Utility.local_utcoffset_minutes(date_utc)
date_local = date_utc + datetime.timedelta(minutes=tz_minutes)
return str(os.path.basename(os.path.dirname(item[0]))) + " (" + date_local.strftime("%c") + ")"
def item_selected(index):
item = items[index]
current_item_ref[0] = os.path.dirname(item[0])
path_label.text = os.path.dirname(item[0])
handle_choose()
self.request_close()
list_widget = Widgets.StringListWidget(ui, items=items, selection_style=Selection.Style.single_or_none, item_getter=stringify_item, border_color="#888", properties={"min-height": 200, "min-width": 560})
list_widget.on_selection_changed = selection_changed
list_widget.on_item_selected = item_selected
list_widget.on_cancel = self.request_close
if len(items) > 0:
list_widget.set_selected_index(0)
items_row = ui.create_row_widget()
items_row.add_spacing(13)
items_row.add(list_widget)
items_row.add_spacing(13)
items_row.add_stretch()
path_row = ui.create_row_widget()
path_row.add_spacing(13)
path_row.add(path_label)
path_row.add_spacing(13)
path_row.add_stretch()
column = ui.create_column_widget()
column.add_spacing(18)
column.add(prompt_row)
column.add_spacing(6)
column.add(explanation1_row)
column.add(explanation2_row)
column.add_spacing(12)
column.add(items_row)
column.add_spacing(6)
column.add(path_row)
column.add_spacing(6)
column.add_stretch()
self.content.add(column)
self.__list_widget = list_widget
def show(self):
super().show()
self.__list_widget.focused = True
if len(items) == 0:
# for initial startup (or with no preferences) try to use a default location
# to avoid the user having to go through the horrendous choose dialog immediately.
try:
documents_dir = self.ui.get_document_location()
workspace_dir = os.path.join(documents_dir, "Nion Swift Library")
Cache.db_make_directory_if_needed(workspace_dir)
path = os.path.join(workspace_dir, "Nion Swift Workspace.nslib")
if not os.path.exists(path):
with open(path, "w") as fp:
json.dump({}, fp)
if os.path.exists(path):
app.switch_library(workspace_dir)
return
except Exception as e:
pass
choose_library_dialog = ChooseLibraryDialog(self.ui, self)
choose_library_dialog.show()
