def test_time_zone_in_extended_data_to_data_element_to_data_item_conversion( self): # test the whole path, redundant? data = numpy.ones((8, 6), numpy.int) metadata = { "description": { "time_zone": { "tz": "+0300", "dst": "+60" } }, "hardware_source": { "one": 1, "two": "b" } } timestamp = datetime.datetime(2013, 11, 18, 14, 5, 4, 1) xdata = DataAndMetadata.new_data_and_metadata(data, metadata=metadata, timestamp=timestamp) data_element = ImportExportManager.create_data_element_from_extended_data( xdata) data_item = ImportExportManager.create_data_item_from_data_element( data_element) self.assertEqual(data_item.metadata["description"]["time_zone"]["tz"], "+0300") self.assertEqual(data_item.metadata["description"]["time_zone"]["dst"], "+60") self.assertEqual("2013-11-18 14:05:04.000001", str(data_item.created))
def test_data_item_to_data_element_and_back_keeps_large_format_flag(self): data_item = DataItem.DataItem(numpy.zeros((4, 4, 4)), large_format=True) with contextlib.closing(data_item): data_element = ImportExportManager.create_data_element_from_data_item(data_item, include_data=True) self.assertTrue(data_element.get("large_format")) with contextlib.closing(ImportExportManager.create_data_item_from_data_element(data_element)) as data_item: self.assertTrue(data_item.large_format)
def test_importing_rgb_does_not_set_large_format(self): data_item = DataItem.DataItem(numpy.zeros((8, 8, 4), dtype=float)) with contextlib.closing(data_item): data_item_rgb = DataItem.DataItem(numpy.zeros((8, 8, 4), dtype=numpy.uint8)) with contextlib.closing(data_item_rgb): data_element = ImportExportManager.create_data_element_from_data_item(data_item, include_data=True) data_element_rgb = ImportExportManager.create_data_element_from_data_item(data_item_rgb, include_data=True) data_element.pop("large_format") data_element_rgb.pop("large_format") with contextlib.closing(ImportExportManager.create_data_item_from_data_element(data_element)) as data_item: with contextlib.closing(ImportExportManager.create_data_item_from_data_element(data_element_rgb)) as data_item_rgb: self.assertTrue(data_item.large_format) self.assertFalse(data_item_rgb.large_format)
def test_data_item_to_data_element_includes_time_zone(self): # created/modified are utc; timezone is specified in metadata/description/time_zone data_item = DataItem.DataItem(numpy.zeros((16, 16))) data_item.created = datetime.datetime(2013, 6, 18, 14, 5, 4, 0) # always utc data_item.timezone = "Europe/Athens" data_item.timezone_offset = "+0300" data_item._set_modified(datetime.datetime(2013, 6, 18, 14, 5, 4, 0)) # always utc data_item.metadata = { "description": { "time_zone": { "tz": "+0300", "dst": "+60" } } } data_element = ImportExportManager.create_data_element_from_data_item( data_item, include_data=False) self.assertEqual( data_element["datetime_modified"], { "dst": "+60", "local_datetime": "2013-06-18T17:05:04", 'tz': "+0300", 'timezone': "Europe/Athens" })
def test_build_table_with_two_display_layers_of_same_calibrated_1d_data( self): with TestContext.create_memory_context() as test_context: calibration = Calibration.Calibration(1.0, 2.0, "nm") document_model = test_context.create_document_model() data_item = DataItem.DataItem(numpy.array([1.1, 1.2, 1.3, 1.4])) data_item.set_dimensional_calibration(0, calibration) data_item.set_intensity_calibration( Calibration.Calibration(0, 1, "e")) data_item2 = DataItem.DataItem(numpy.array([2.1, 2.2, 2.3, 2.4])) data_item2.set_dimensional_calibration(0, calibration) data_item2.set_intensity_calibration( Calibration.Calibration(0, 1, "e")) document_model.append_data_item(data_item) document_model.append_data_item(data_item2) display_item = document_model.get_display_item_for_data_item( data_item) display_item.append_display_data_channel_for_data_item(data_item2) display_item._set_display_layer_property(0, "label", "W") display_item._set_display_layer_property(1, "label", "T") headers, data = ImportExportManager.build_table(display_item) self.assertEqual(3, len(headers)) self.assertEqual(3, len(data)) self.assertEqual("X (nm)", headers[0]) self.assertEqual("W (e)", headers[1]) self.assertEqual("T (e)", headers[2]) self.assertTrue( numpy.array_equal( calibration.convert_to_calibrated_value( numpy.arange(0, data_item.data.shape[0])), data[0])) self.assertTrue(numpy.array_equal(data_item.data, data[1])) self.assertTrue(numpy.array_equal(data_item2.data, data[2]))
def do_export(self, display_items): directory = self.directory writer = self.writer if directory: for index, display_item in enumerate(display_items): data_item = display_item.data_item try: components = list() if self.options.get("prefix", False): components.append(str(self.prefix_edit_widget.text)) if self.options.get("title", False): title = unicodedata.normalize('NFKC', data_item.title) title = re.sub('[^\w\s-]', '', title, flags=re.U).strip() title = re.sub('[-\s]+', '-', title, flags=re.U) components.append(title) if self.options.get("date", False): components.append(data_item.created_local.isoformat().replace(':', '')) if self.options.get("dimensions", False): components.append( "x".join([str(shape_n) for shape_n in data_item.dimensional_shape])) if self.options.get("sequence", False): components.append(str(index)) filename = "_".join(components) extension = writer.extensions[0] path = os.path.join(directory, "{0}.{1}".format(filename, extension)) ImportExportManager.ImportExportManager().write_display_item_with_writer(self.ui, writer, display_item, path) except Exception as e: logging.debug("Could not export image %s / %s", str(data_item), str(e)) traceback.print_exc() traceback.print_stack()
def exec_grab(): # this will execute in a thread; the enclosing async routine will continue when it finishes try: start_time = time.time() max_wait_time = max( hardware_source.get_current_frame_time() * 1.5, 3) while hardware_source.is_playing: if time.time() - start_time > max_wait_time: success_ref[0] = False return time.sleep(0.01) data_element_groups = hardware_source.get_buffer_data( -frame_count, frame_count) for data_element_group in data_element_groups: if self.cancel_event.is_set(): success_ref[0] = False break xdata_group = list() for data_element in data_element_group: xdata = ImportExportManager.convert_data_element_to_data_and_metadata( data_element) xdata_group.append(xdata) xdata_group_list.append(xdata_group) self.progress_model.value = 100 except Exception as e: import traceback traceback.print_exc() success_ref[0] = False
def acquire_stack_and_sum(number_frames, energy_offset_per_frame, document_controller, final_layout_fn): # grab the document model and workspace for convenience with document_controller.create_task_context_manager( _("Multiple Shift EELS Acquire"), "table") as task: # acquire the stack. it will be added to the document by queueing to the main thread at the end of this method. stack_data_item = acquire_series(number_frames, energy_offset_per_frame, task) stack_data_item.title = _("Spectrum Stack") # align and sum the stack data_element = dict() summed_image, shifts = align_stack(stack_data_item.data, task) # add the summed image to Swift data_element["data"] = summed_image data_element["title"] = "Aligned and summed spectra" # strip off the first dimension that we sum over for dimensional_calibration in stack_data_item.dimensional_calibrations[ 1:]: data_element.setdefault( "spatial_calibrations", list()).append({ "origin": dimensional_calibration.offset, # TODO: fix me "scale": dimensional_calibration.scale, "units": dimensional_calibration.units }) # set the energy dispersive calibration so that the ZLP is at zero eV zlp_position_pixels = numpy.sum(summed_image, axis=0).argmax() zlp_position_calibrated_units = -zlp_position_pixels * data_element[ "spatial_calibrations"][1]["scale"] data_element["spatial_calibrations"][1][ "offset"] = zlp_position_calibrated_units sum_data_item = ImportExportManager.create_data_item_from_data_element( data_element) dispersive_sum = numpy.sum(summed_image, axis=1) differential = numpy.diff(dispersive_sum) top = numpy.argmax(differential) bottom = numpy.argmin(differential) _midpoint = numpy.mean([bottom, top]) / dispersive_sum.shape[0] _integration_width = float( numpy.abs(bottom - top)) / dispersive_sum.shape[ 0] #* data_element["spatial_calibrations"][0]["scale"] document_controller.queue_task(final_layout_fn) document_controller.queue_task( functools.partial(show_in_panel, stack_data_item, document_controller, "multiple_shift_eels_stack")) document_controller.queue_task( functools.partial( show_in_panel, sum_data_item, document_controller, "multiple_shift_eels_aligned_summed_stack")) document_controller.queue_task( functools.partial(add_line_profile, sum_data_item, document_controller, "multiple_shift_eels_spectrum", _midpoint, _integration_width))
def test_data_element_date_gets_set_as_data_item_created_date(self): data_element = dict() data_element["version"] = 1 data_element["data"] = numpy.zeros((16, 16), dtype=numpy.double) data_element["datetime_modified"] = { 'tz': '+0300', 'dst': '+60', 'local_datetime': '2015-06-10T19:31:52.780511' } data_item = ImportExportManager.create_data_item_from_data_element( data_element) self.assertIsNotNone(data_item.created) self.assertEqual(data_item.timezone_offset, "+0300") local_offset_seconds = int( round((datetime.datetime.now() - datetime.datetime.utcnow()).total_seconds())) # check both matches for DST match1 = datetime.datetime(year=2015, month=6, day=10, hour=19 - 3, minute=31, second=52, microsecond=780511) + datetime.timedelta( seconds=local_offset_seconds) match2 = datetime.datetime(year=2015, month=6, day=10, hour=19 - 3, minute=31, second=52, microsecond=780511) + datetime.timedelta( seconds=local_offset_seconds + 3600) self.assertTrue(data_item.created_local == match1 or data_item.created_local == match2)
def test_csv1_exporter_handles_multi_layer_display_item_with_different_calibration(self): with TestContext.create_memory_context() as test_context: document_model = test_context.create_document_model() data_item = DataItem.DataItem(numpy.full((50, ), 0, dtype=numpy.uint32)) data_item.dimensional_calibrations = [Calibration.Calibration(offset=0, scale=1, units="eV")] data_item.intensity_calibration = Calibration.Calibration(offset=10, scale=2) data_item2 = DataItem.DataItem(numpy.full((100, ), 1, dtype=numpy.uint32)) data_item2.dimensional_calibrations = [Calibration.Calibration(offset=-10, scale=2, units="eV")] data_item2.intensity_calibration = Calibration.Calibration(offset=5, scale=2) document_model.append_data_item(data_item) document_model.append_data_item(data_item2, False) display_item = document_model.get_display_item_for_data_item(data_item) display_item.display_type = "line_plot" display_item.append_display_data_channel_for_data_item(data_item2) current_working_directory = os.getcwd() file_path = os.path.join(current_working_directory, "__file.csv") handler = ImportExportManager.CSV1ImportExportHandler("csv1-io-handler", "CSV 1D", ["csv"]) handler.write_display_item(None, display_item, file_path, "csv") self.assertTrue(os.path.exists(file_path)) try: saved_data = numpy.genfromtxt(file_path, delimiter=", ") self.assertSequenceEqual(saved_data.shape, (max(data_item.xdata.data_shape[0], data_item2.xdata.data_shape[0]), 4)) self.assertTrue(numpy.allclose(saved_data[:50, 0], numpy.linspace(0, 49, 50))) self.assertTrue(numpy.allclose(saved_data[:50, 1], 10)) self.assertTrue(numpy.allclose(saved_data[:, 2], numpy.linspace(-10, 188, 100))) self.assertTrue(numpy.allclose(saved_data[:, 3], 7)) finally: os.remove(file_path)
def create_result_data_item( self, data_dict: typing.Mapping[str, typing.Any]) -> None: reset_color_cycle() display_item = None sorted_indices = numpy.argsort( [parms['start_ev'] for parms in data_dict['parameter_list']]) display_layer_index = 0 for i in sorted_indices: index = data_dict['parameter_list'][i]['index'] xdata = ImportExportManager.convert_data_element_to_data_and_metadata( data_dict['data_element_list'][i]) start_ev = data_dict['parameter_list'][i]['start_ev'] end_ev = data_dict['parameter_list'][i]['end_ev'] number_frames = data_dict['parameter_list'][i]['frames'] exposure_ms = data_dict['parameter_list'][i]['exposure_ms'] summed = ' (summed)' if not data_dict['data_element_list'][i].get( 'is_sequence', False) and number_frames > 1 else '' data_item = None if i == sorted_indices[ 0] and xdata.datum_dimension_count == 1 and data_dict[ 'settings_list'][i]['use_multi_eels_calibration']: data_item = self.document_controller.library.create_data_item_from_data_and_metadata( xdata, title='MultiAcquire (stacked)') display_item = self.__api.library._document_model.get_display_item_for_data_item( data_item._data_item) display_layer_index += 1 # display item has one display layer already #else: units_str = ' eV' if data_dict['settings_list'][i][ 'use_multi_eels_calibration'] else '' new_data_item = self.document_controller.library.create_data_item_from_data_and_metadata( xdata, title='MultiAcquire #{:d}, {:g}-{:g}{}, {:g}x{:g} ms{}'.format( index + 1, start_ev, end_ev, units_str, number_frames, exposure_ms, summed)) data_item_metadata = new_data_item.metadata metadata = dict( data_item_metadata) if data_item_metadata else dict() metadata['MultiAcquire.parameters'] = data_dict['parameter_list'][ i] metadata['MultiAcquire.settings'] = data_dict['settings_list'][i] new_data_item.set_metadata(metadata) if display_item: display_item.append_display_data_channel_for_data_item( data_item._data_item if data_item else new_data_item. _data_item) start_ev = data_dict['parameter_list'][i]['start_ev'] end_ev = data_dict['parameter_list'][i]['end_ev'] display_layer_label = '#{:d}: {:g}-{:g}{}, {:g}x{:g} ms{}'.format( index + 1, start_ev, end_ev, units_str, number_frames, exposure_ms, summed) display_item._set_display_layer_properties( display_layer_index, label=display_layer_label, stroke_color=get_next_color(), fill_color=None) display_layer_index += 1 if display_item: display_item.set_display_property('legend_position', 'top-right') display_item.title = 'MultiAcquire (stacked)' show_display_item(self.document_controller, display_item)
def test_data_element_with_uuid_assigns_uuid_to_data_item(self): data_element = dict() data_element["version"] = 1 data_element["data"] = numpy.zeros((16, 16), dtype=numpy.double) data_element_uuid = uuid.uuid4() data_element["uuid"] = str(data_element_uuid) with contextlib.closing(ImportExportManager.create_data_item_from_data_element(data_element)) as data_item: self.assertEqual(data_item.uuid, data_element_uuid)
def test_convert_data_element_records_time_zone_in_data_item_metadata(self): data_element = dict() data_element["version"] = 1 data_element["data"] = numpy.zeros((16, 16), dtype=numpy.double) data_element["datetime_modified"] = Utility.get_current_datetime_item() with contextlib.closing(ImportExportManager.create_data_item_from_data_element(data_element)) as data_item: self.assertIsNotNone(data_item.created) self.assertEqual(data_item.timezone_offset, data_element["datetime_modified"]["tz"])
def test_data_element_to_extended_data_includes_time_zone(self): data_element = dict() data_element["version"] = 1 data_element["data"] = numpy.zeros((16, 16), dtype=numpy.double) data_element["datetime_modified"] = {'tz': '+0300', 'dst': '+60', 'local_datetime': '2015-06-10T19:31:52.780511'} xdata = ImportExportManager.convert_data_element_to_data_and_metadata(data_element) self.assertEqual(xdata.timezone_offset, "+0300") self.assertEqual(str(xdata.timestamp), "2015-06-10 16:31:52.780511")
def test_data_element_to_data_item_includes_time_zone(self): data_element = dict() data_element["version"] = 1 data_element["data"] = numpy.zeros((16, 16), dtype=numpy.double) data_element["datetime_modified"] = {'tz': '+0300', 'dst': '+60', 'local_datetime': '2015-06-10T19:31:52.780511'} with contextlib.closing(ImportExportManager.create_data_item_from_data_element(data_element)) as data_item: self.assertEqual(data_item.timezone_offset, "+0300") self.assertEqual(str(data_item.created), "2015-06-10 16:31:52.780511")
def test_extended_data_to_data_element_includes_time_zone(self): # extended data timestamp is utc; timezone is specified in metadata/description/time_zone data = numpy.ones((8, 6), int) metadata = {"description": {"time_zone": {"tz": "+0300", "dst": "+60"}}} timestamp = datetime.datetime(2013, 11, 18, 14, 5, 4, 0) xdata = DataAndMetadata.new_data_and_metadata(data, metadata=metadata, timestamp=timestamp) data_element = ImportExportManager.create_data_element_from_extended_data(xdata) self.assertEqual(data_element["datetime_modified"], {"dst": "+60", "local_datetime": "2013-11-18T17:05:04", 'tz': "+0300"})
def test_convert_data_element_sets_timezone_and_timezone_offset_if_present(self): data_element = dict() data_element["version"] = 1 data_element["data"] = numpy.zeros((16, 16), dtype=numpy.double) data_element["datetime_modified"] = {'tz': '+0300', 'dst': '+60', 'local_datetime': '2015-06-10T19:31:52.780511', 'timezone': 'Europe/Athens'} with contextlib.closing(ImportExportManager.create_data_item_from_data_element(data_element)) as data_item: self.assertIsNotNone(data_item.created) self.assertEqual(data_item.timezone, "Europe/Athens") self.assertEqual(data_item.timezone_offset, "+0300")
def test_data_item_with_numpy_bool_to_data_element_produces_json_compatible_dict( self): data_item = DataItem.DataItem(numpy.zeros((16, 16))) data_item.large_format = numpy.product( (2, 3, 4), dtype=numpy.int64) > 10 # produces a numpy.bool_ with contextlib.closing(data_item): data_element = ImportExportManager.create_data_element_from_data_item( data_item, include_data=False) json.dumps(data_element)
def create_result_data_item(self, data_dict): display_layers = [] reset_color_cycle() display_item = None sorted_indices = numpy.argsort( [parms['start_ev'] for parms in data_dict['parameter_list']]) for i in sorted_indices: index = data_dict['parameter_list'][i]['index'] xdata = ImportExportManager.convert_data_element_to_data_and_metadata( data_dict['data_element_list'][i]) start_ev = data_dict['parameter_list'][i]['start_ev'] end_ev = data_dict['parameter_list'][i]['end_ev'] number_frames = data_dict['parameter_list'][i]['frames'] exposure_ms = data_dict['parameter_list'][i]['exposure_ms'] summed = ' (summed)' if not data_dict['data_element_list'][i].get( 'is_sequence', False) and number_frames > 1 else '' data_item = None if i == sorted_indices[0] and xdata.datum_dimension_count == 1: data_item = self.document_controller.library.create_data_item_from_data_and_metadata( xdata, title='MultiAcquire (stacked)') display_item = self.__api.library._document_model.get_display_item_for_data_item( data_item._data_item) #new_data_item = data_item #else: new_data_item = self.document_controller.library.create_data_item_from_data_and_metadata( xdata, title='MultiAcquire #{:d}, {:g}-{:g} eV, {:g}x{:g} ms{}'. format(index + 1, start_ev, end_ev, number_frames, exposure_ms, summed)) metadata = new_data_item.metadata metadata['MultiAcquire.parameters'] = data_dict['parameter_list'][ i] metadata['MultiAcquire.settings'] = data_dict['settings_list'][i] new_data_item.set_metadata(metadata) if display_item: display_item.append_display_data_channel_for_data_item( data_item._data_item if data_item else new_data_item. _data_item) start_ev = data_dict['parameter_list'][i]['start_ev'] end_ev = data_dict['parameter_list'][i]['end_ev'] display_layers.append({ 'label': '#{:d}: {:g}-{:g} eV, {:g}x{:g} ms{}'.format( index + 1, start_ev, end_ev, number_frames, exposure_ms, summed), 'data_index': len(display_layers), 'stroke_color': get_next_color(), 'fill_color': None }) if display_item: display_item.display_layers = display_layers display_item.set_display_property('legend_position', 'top-right') display_item.title = 'MultiAcquire (stacked)' show_display_item(self.document_controller, display_item)
def test_data_element_to_extended_data_conversion(self): data = numpy.ones((8, 6), int) intensity_calibration = Calibration.Calibration(offset=1, scale=1.1, units="one") dimensional_calibrations = [Calibration.Calibration(offset=2, scale=2.1, units="two"), Calibration.Calibration(offset=3, scale=2.2, units="two")] metadata = {"hardware_source": {"one": 1, "two": "b"}} timestamp = datetime.datetime.now() data_descriptor = DataAndMetadata.DataDescriptor(is_sequence=False, collection_dimension_count=1, datum_dimension_count=1) xdata = DataAndMetadata.new_data_and_metadata(data, intensity_calibration=intensity_calibration, dimensional_calibrations=dimensional_calibrations, metadata=metadata, timestamp=timestamp, data_descriptor=data_descriptor) data_element = ImportExportManager.create_data_element_from_extended_data(xdata) new_xdata = ImportExportManager.convert_data_element_to_data_and_metadata(data_element) self.assertTrue(numpy.array_equal(data, new_xdata.data)) self.assertNotEqual(id(new_xdata.intensity_calibration), id(intensity_calibration)) self.assertEqual(new_xdata.intensity_calibration, intensity_calibration) self.assertNotEqual(id(new_xdata.dimensional_calibrations[0]), id(dimensional_calibrations[0])) self.assertEqual(new_xdata.dimensional_calibrations, dimensional_calibrations) self.assertNotEqual(id(new_xdata.metadata), id(metadata)) self.assertEqual(new_xdata.metadata, metadata) self.assertNotEqual(id(new_xdata.data_descriptor), id(data_descriptor)) self.assertEqual(new_xdata.data_descriptor, data_descriptor)
def test_data_item_to_data_element_produces_json_compatible_dict(self): data_item = DataItem.DataItem(numpy.zeros((16, 16))) data_item.created = datetime.datetime(2013, 6, 18, 14, 5, 4, 0) # always utc data_item.timezone = "Europe/Athens" data_item.timezone_offset = "+0300" data_item.source_file_path = "/path/to/source/file" data_item._set_modified(datetime.datetime(2013, 6, 18, 14, 5, 4, 0)) # always utc data_item.metadata = {"description": {"time_zone": {"tz": "+0300", "dst": "+60"}}} data_element = ImportExportManager.create_data_element_from_data_item(data_item, include_data=False) json.dumps(data_element)
def test_sub_area_size_change(self): document_model = DocumentModel.DocumentModel() with contextlib.closing(document_model): data_element = dict() data_element["version"] = 1 data_element["data"] = numpy.zeros((16, 16), dtype=numpy.double) data_item = ImportExportManager.create_data_item_from_data_element(data_element) document_model.append_data_item(data_item) self.assertEqual(data_item.dimensional_shape, (16, 16)) self.assertEqual(data_item.data_dtype, numpy.double) data_element["data"] = numpy.zeros((8, 8), dtype=numpy.double) data_element["sub_area"] = ((0,0), (4, 8)) ImportExportManager.update_data_item_from_data_element(data_item, data_element) self.assertEqual(data_item.dimensional_shape, (8, 8)) self.assertEqual(data_item.data_dtype, numpy.double) data_element["data"] = numpy.zeros((8, 8), dtype=numpy.float) data_element["sub_area"] = ((0,0), (4, 8)) ImportExportManager.update_data_item_from_data_element(data_item, data_element) self.assertEqual(data_item.dimensional_shape, (8, 8)) self.assertEqual(data_item.data_dtype, numpy.float)
def test_sub_area_size_change(self): with TestContext.create_memory_context() as test_context: document_model = test_context.create_document_model() data_element = dict() data_element["version"] = 1 data_element["data"] = numpy.zeros((16, 16), dtype=numpy.double) data_item = ImportExportManager.create_data_item_from_data_element(data_element) document_model.append_data_item(data_item) self.assertEqual(data_item.dimensional_shape, (16, 16)) self.assertEqual(data_item.data_dtype, numpy.double) data_element["data"] = numpy.zeros((8, 8), dtype=numpy.double) data_element["sub_area"] = ((0,0), (4, 8)) ImportExportManager.update_data_item_from_data_element(data_item, data_element) self.assertEqual(data_item.dimensional_shape, (8, 8)) self.assertEqual(data_item.data_dtype, numpy.double) data_element["data"] = numpy.zeros((8, 8), dtype=float) data_element["sub_area"] = ((0,0), (4, 8)) ImportExportManager.update_data_item_from_data_element(data_item, data_element) self.assertEqual(data_item.dimensional_shape, (8, 8)) self.assertEqual(data_item.data_dtype, float)
def test_creating_data_element_with_sequence_and_implicit_datum_size_data_makes_correct_data_item(self): data_element = dict() data_element["version"] = 1 data_element["data"] = numpy.zeros((4, 16, 16), dtype=numpy.double) data_element["is_sequence"] = True with contextlib.closing(ImportExportManager.create_data_item_from_data_element(data_element)) as data_item: self.assertEqual(data_item.is_sequence, True) self.assertEqual(data_item.collection_dimension_count, 0) self.assertEqual(data_item.datum_dimension_count, 2) self.assertEqual(data_item.xdata.is_sequence, True) self.assertEqual(data_item.xdata.collection_dimension_count, 0) self.assertEqual(data_item.xdata.datum_dimension_count, 2)
def test_importing_large_numpy_file_sets_large_format_flag(self): current_working_directory = os.getcwd() file_path_npy = os.path.join(current_working_directory, "__file.npy") numpy.save(file_path_npy, numpy.zeros((4, 4, 4))) handler = ImportExportManager.NumPyImportExportHandler( "numpy-io-handler", "npy", ["npy"]) try: data_items = handler.read_data_items(None, "npy", file_path_npy) self.assertEqual(len(data_items), 1) data_item = data_items[0] self.assertTrue(data_item.large_format) finally: os.remove(file_path_npy)
def test_creating_data_element_with_sequence_data_makes_correct_data_item(self): data_element = dict() data_element["version"] = 1 data_element["data"] = numpy.zeros((4, 16, 16), dtype=numpy.double) data_element["is_sequence"] = True data_element["collection_dimension_count"] = 0 data_element["datum_dimension_count"] = 2 data_item = ImportExportManager.create_data_item_from_data_element(data_element) self.assertEqual(data_item.is_sequence, True) self.assertEqual(data_item.collection_dimension_count, 0) self.assertEqual(data_item.datum_dimension_count, 2) self.assertEqual(data_item.xdata.is_sequence, True) self.assertEqual(data_item.xdata.collection_dimension_count, 0) self.assertEqual(data_item.xdata.datum_dimension_count, 2)
def grab_sequence( self, count: int, **kwargs ) -> typing.Optional[typing.List[DataAndMetadata.DataAndMetadata]]: self.start_playing() frames = self.acquire_sequence(count) if frames is not None: xdatas = list() for data_element in frames: data_element["is_sequence"] = True data_element["collection_dimension_count"] = 0 data_element["datum_dimension_count"] = len( data_element["data"].shape) - 1 xdata = ImportExportManager.convert_data_element_to_data_and_metadata( data_element) xdatas.append(xdata) return xdatas return None
def test_ndata_write_to_then_read_from_temp_file(self): document_model = DocumentModel.DocumentModel() with contextlib.closing(document_model): current_working_directory = os.getcwd() file_path = os.path.join(current_working_directory, "__file.ndata") handler = ImportExportManager.NDataImportExportHandler("ndata1-io-handler", "ndata", ["ndata"]) data_item = DataItem.DataItem(numpy.zeros((16, 16), dtype=numpy.double)) document_model.append_data_item(data_item) display_item = document_model.get_display_item_for_data_item(data_item) handler.write_display_item(None, display_item, file_path, "ndata") self.assertTrue(os.path.exists(file_path)) try: data_items = handler.read_data_items(None, "ndata", file_path) self.assertEqual(len(data_items), 1) data_item = data_items[0] finally: os.remove(file_path)
def test_npy_write_to_then_read_from_temp_file(self): document_model = DocumentModel.DocumentModel() with contextlib.closing(document_model): current_working_directory = os.getcwd() file_path_npy = os.path.join(current_working_directory, "__file.npy") file_path_json = os.path.join(current_working_directory, "__file.json") numpy.save(file_path_npy, numpy.zeros((16, 16))) with open(file_path_json, "w") as f: json.dump({"version": 1}, f) handler = ImportExportManager.NumPyImportExportHandler("numpy-io-handler", "npy", ["npy"]) try: data_items = handler.read_data_items(None, "npy", file_path_npy) self.assertEqual(len(data_items), 1) data_item = data_items[0] finally: os.remove(file_path_npy) os.remove(file_path_json)
def grab_partial( self, *, update_period: float = 1.0) -> typing.Tuple[bool, bool, int]: # updates the full readout data, returns a tuple of is complete, is canceled, and # the number of valid rows. start = self.__start frame_parameters = self.__camera_frame_parameters exposure = frame_parameters.exposure_ms / 1000.0 n = min(max(int(update_period / exposure), 1), self.__count - start) is_complete = start + n == self.__count # print(f"{start=} {n=} {self.__count=} {is_complete=}") data_element = self.__camera_device._acquire_sequence((n)) if data_element and not self.__aborted: xdata = ImportExportManager.convert_data_element_to_data_and_metadata( data_element) dimensional_calibrations = tuple( Calibration.Calibration() for _ in range(len(self.__collection_shape))) + tuple( xdata.dimensional_calibrations[1:]) assert self.__xdata self.__xdata._set_intensity_calibration( xdata.intensity_calibration) self.__xdata._set_dimensional_calibrations( dimensional_calibrations) if len(self.__collection_shape) > 1: row_size = self.__collection_shape[-1] start_row = start // row_size rows = n // row_size metadata = dict(xdata.metadata) metadata.setdefault("hardware_source", dict())["valid_rows"] = start_row + rows self.__xdata._set_metadata(metadata) else: row_size = 1 # convert from a sequence to a collection. assert self.__data is not None self.__data.reshape( (self.__data.shape[0] * row_size, ) + self.__data.shape[len(self.__collection_shape):])[ start:start + n, ...] = xdata._data_ex self.__start = start + n return is_complete, False, start + n self.__start = 0 return True, True, 0