def create_data_element_from_extended_data(xdata: DataAndMetadata.DataAndMetadata) -> dict: data_element = dict() data_element["version"] = 1 data_element["reader_version"] = 1 data_element["data"] = xdata.data data_element["large_format"] = len(xdata.dimensional_shape) > 2 dimensional_calibrations = xdata.dimensional_calibrations calibrations_element = list() for calibration in dimensional_calibrations: calibration_element = { "offset": calibration.offset, "scale": calibration.scale, "units": calibration.units } calibrations_element.append(calibration_element) data_element["spatial_calibrations"] = calibrations_element intensity_calibration = xdata.intensity_calibration intensity_calibration_element = { "offset": intensity_calibration.offset, "scale": intensity_calibration.scale, "units": intensity_calibration.units } data_element["intensity_calibration"] = intensity_calibration_element if xdata.is_sequence: data_element["is_sequence"] = xdata.is_sequence data_element["collection_dimension_count"] = xdata.collection_dimension_count data_element["datum_dimension_count"] = xdata.datum_dimension_count data_element["metadata"] = copy.deepcopy(xdata.metadata) # properties is redundant; but here for backwards compatibility data_element["properties"] = copy.deepcopy(xdata.metadata.get("hardware_source", dict())) tz_minutes = None dst_minutes = None timezone = None time_zone_dict = xdata.metadata.get("description", dict()).get("time_zone") if time_zone_dict: # note: dst is informational only; tz already include dst tz_value = time_zone_dict["tz"] tz_minutes = (int(tz_value[1:3]) * 60 + int(tz_value[3:5])) * (-1 if tz_value[0] == '-' else 1) dst_minutes = int(time_zone_dict["dst"]) timezone = time_zone_dict.get("timezone") data_element["datetime_modified"] = Utility.get_datetime_item_from_utc_datetime(xdata.timestamp, tz_minutes, dst_minutes, timezone) data_element["datetime_original"] = Utility.get_datetime_item_from_utc_datetime(xdata.timestamp, tz_minutes, dst_minutes, timezone) return data_element
def write_properties(self, properties: PersistentDictType, file_datetime: datetime.datetime) -> None: """ Write properties to the ndata file specified by reference. :param reference: the reference to which to write :param properties: the dict to write to the file :param file_datetime: the datetime for the file The properties param must not change during this method. Callers should take care to ensure this does not happen. """ with self.__lock: absolute_file_path = self.__file_path #logging.debug("WRITE properties %s for %s", absolute_file_path, key) make_directory_if_needed(os.path.dirname(absolute_file_path)) exists = os.path.exists(absolute_file_path) if exists: rewrite_zip(absolute_file_path, Utility.clean_dict(properties)) else: write_zip(absolute_file_path, None, Utility.clean_dict(properties)) # convert to utc time. tz_minutes = Utility.local_utcoffset_minutes(file_datetime) timestamp = calendar.timegm( file_datetime.timetuple()) - tz_minutes * 60 os.utime(absolute_file_path, (time.time(), timestamp))
def test_reset_display_limits_on_various_value_types_write_to_clean_json( self): with TestContext.create_memory_context() as test_context: document_model = test_context.create_document_model() dtypes = (numpy.float32, float, numpy.complex64, numpy.complex128, numpy.int16, numpy.uint16, numpy.int32, numpy.uint32) for dtype in dtypes: data_item = DataItem.DataItem(numpy.ones((16, 16), dtype)) document_model.append_data_item(data_item) display_item = document_model.get_display_item_for_data_item( data_item) display_item.display_data_channels[0].reset_display_limits() Utility.clean_dict(data_item.properties)
def test_reset_display_limits_on_various_value_types_write_to_clean_json( self): document_model = DocumentModel.DocumentModel() with contextlib.closing(document_model): dtypes = (numpy.float32, numpy.float64, numpy.complex64, numpy.complex128, numpy.int16, numpy.uint16, numpy.int32, numpy.uint32) for dtype in dtypes: data_item = DataItem.DataItem(numpy.ones((16, 16), dtype)) document_model.append_data_item(data_item) display_item = document_model.get_display_item_for_data_item( data_item) display_item.display_data_channels[0].reset_display_limits() Utility.clean_dict(data_item.properties)
def get_interactive(self, version=None): actual_version = "1.0.0" if Utility.compare_versions(version, actual_version) > 0: raise NotImplementedError( "API requested version %s is greater than %s." % (version, actual_version)) return interactive_session
def get_ui(self, version): actual_version = "1.0.0" if Utility.compare_versions(version, actual_version) > 0: raise NotImplementedError( "API requested version %s is greater than %s." % (version, actual_version)) return Declarative.DeclarativeUI()
def create_data_element_from_data_item(data_item, include_data=True): data_element = dict() data_element["version"] = 1 data_element["reader_version"] = 1 if data_item.has_data: data_element["large_format"] = data_item.large_format if include_data: data_element["data"] = data_item.data dimensional_calibrations = data_item.dimensional_calibrations if dimensional_calibrations is not None: calibrations_element = list() for calibration in dimensional_calibrations: calibration_element = { "offset": calibration.offset, "scale": calibration.scale, "units": calibration.units } calibrations_element.append(calibration_element) data_element["spatial_calibrations"] = calibrations_element intensity_calibration = data_item.intensity_calibration if intensity_calibration is not None: intensity_calibration_element = { "offset": intensity_calibration.offset, "scale": intensity_calibration.scale, "units": intensity_calibration.units } data_element["intensity_calibration"] = intensity_calibration_element if data_item.is_sequence: data_element["is_sequence"] = data_item.is_sequence data_element["collection_dimension_count"] = data_item.collection_dimension_count data_element["datum_dimension_count"] = data_item.datum_dimension_count data_element["metadata"] = copy.deepcopy(data_item.metadata) data_element["properties"] = copy.deepcopy(data_item.metadata.get("hardware_source", dict())) data_element["title"] = data_item.title data_element["source_file_path"] = data_item.source_file_path.as_posix() if data_item.source_file_path else None tz_value = data_item.timezone_offset timezone = data_item.timezone dst_minutes = None time_zone_dict = data_item.metadata.get("description", dict()).get("time_zone") if time_zone_dict: # note: dst is informational only; tz already include dst if tz_value is None: tz_value = time_zone_dict["tz"] dst_minutes = int(time_zone_dict["dst"]) if timezone is None: timezone = time_zone_dict.get("timezone") if tz_value is not None: tz_minutes = (int(tz_value[1:3]) * 60 + int(tz_value[3:5])) * (-1 if tz_value[0] == '-' else 1) data_element["datetime_modified"] = Utility.get_datetime_item_from_utc_datetime(data_item.created, tz_minutes, dst_minutes, timezone) data_element["datetime_original"] = Utility.get_datetime_item_from_utc_datetime(data_item.created, tz_minutes, dst_minutes, timezone) data_element["uuid"] = str(data_item.uuid) # operation # graphics return data_element
def test_convert_data_element_records_time_zone_in_data_item_metadata(self): data_element = dict() data_element["version"] = 1 data_element["data"] = numpy.zeros((16, 16), dtype=numpy.double) data_element["datetime_modified"] = Utility.get_current_datetime_item() with contextlib.closing(ImportExportManager.create_data_item_from_data_element(data_element)) as data_item: self.assertIsNotNone(data_item.created) self.assertEqual(data_item.timezone_offset, data_element["datetime_modified"]["tz"])
def __init__(self, storage_system, storage_handler, properties): self.__storage_system = storage_system self.__storage_handler = storage_handler self.__properties = Migration.transform_to_latest( Utility.clean_dict( copy.deepcopy(properties) if properties else dict())) self.__properties_lock = threading.RLock() self.__write_delayed = False
def get_api(self, version): actual_version = "1.0.0" if Utility.compare_versions(version, actual_version) > 0: raise NotImplementedError("Camera API requested version %s is greater than %s." % (version, actual_version)) class CameraFacade: def __init__(self): pass return CameraFacade()
def _repaint(self, drawing_context: DrawingContext.DrawingContext) -> None: super()._repaint(drawing_context) with self.__drawing_context_lock: drawing_context.add(self.__drawing_context) if self.__display_frame_rate_id: Utility.fps_tick("display_" + self.__display_frame_rate_id) if self.__display_frame_rate_id: fps = Utility.fps_get("display_" + self.__display_frame_rate_id) fps2 = Utility.fps_get("frame_" + self.__display_frame_rate_id) fps3 = Utility.fps_get("update_" + self.__display_frame_rate_id) canvas_bounds = self.canvas_bounds if canvas_bounds: with drawing_context.saver(): font = "normal 11px serif" text_pos = canvas_bounds.top_right + Geometry.IntPoint( y=0, x=-100) drawing_context.begin_path() drawing_context.move_to(text_pos.x, text_pos.y) drawing_context.line_to(text_pos.x + 120, text_pos.y) drawing_context.line_to(text_pos.x + 120, text_pos.y + 60) drawing_context.line_to(text_pos.x, text_pos.y + 60) drawing_context.close_path() drawing_context.fill_style = "rgba(255, 255, 255, 0.6)" drawing_context.fill() drawing_context.font = font drawing_context.text_baseline = "middle" drawing_context.text_align = "left" drawing_context.fill_style = "#000" drawing_context.fill_text("display:" + fps, text_pos.x + 8, text_pos.y + 10) drawing_context.fill_text("frame:" + fps2, text_pos.x + 8, text_pos.y + 30) drawing_context.fill_text("update:" + fps3, text_pos.x + 8, text_pos.y + 50)
def __write_properties(self, object): if self.__write_delay_counts.get(object, 0) == 0: persistent_object_parent = object.persistent_object_parent if object else None if object and isinstance(object, DataItem.DataItem): self.__get_storage_for_item(object).rewrite_item(object) elif not persistent_object_parent: if self.__filepath: # atomically overwrite filepath = pathlib.Path(self.__filepath) temp_filepath = filepath.with_name(filepath.name + ".temp") with temp_filepath.open("w") as fp: json.dump(Utility.clean_dict(self.__properties), fp) os.replace(temp_filepath, filepath) else: self.__write_properties(persistent_object_parent.parent)
def auto_migrate_data_item(reader_info, persistent_storage_system, new_persistent_storage_system, index: int, count: int) -> None: storage_handler = reader_info.storage_handler properties = reader_info.properties properties = Utility.clean_dict( copy.deepcopy(properties) if properties else dict()) data_item_uuid = uuid.UUID(properties["uuid"]) if persistent_storage_system == new_persistent_storage_system: if reader_info.changed_ref[0]: storage_handler.write_properties( Migration.transform_from_latest(copy.deepcopy(properties)), datetime.datetime.now()) persistent_storage_system.register_data_item(None, data_item_uuid, storage_handler, properties) else: # create a temporary data item that can be used to get the new file reference old_data_item = DataItem.DataItem(item_uuid=data_item_uuid) old_data_item.begin_reading() old_data_item.read_from_dict(properties) old_data_item.finish_reading() old_data_item_path = storage_handler.reference # ask the storage system for the file handler for the data item path file_handler = new_persistent_storage_system.get_file_handler_for_file( old_data_item_path) # ask the storage system to make a storage handler (an instance of a file handler) for the data item # this ensures that the storage handler (file format) is the same as before. target_storage_handler = new_persistent_storage_system.make_storage_handler( old_data_item, file_handler) if target_storage_handler: os.makedirs(os.path.dirname(target_storage_handler.reference), exist_ok=True) shutil.copyfile(storage_handler.reference, target_storage_handler.reference) target_storage_handler.write_properties( Migration.transform_from_latest(copy.deepcopy(properties)), datetime.datetime.now()) new_persistent_storage_system.register_data_item( None, data_item_uuid, target_storage_handler, properties) logging.getLogger("migration").info( "Copying data item ({}/{}) {} to new library.".format( index + 1, count, data_item_uuid)) else: logging.getLogger("migration").warning( "Unable to copy data item %s to new library.".format( data_item_uuid))
def read_data_elements(self, ui, extension, path): data = None try: data = Image.read_image_from_file(ui, path) except Exception as e: pass if data is not None: data_element = dict() data_element["version"] = 1 data_element["data"] = data if os.path.exists(path) or path.startswith(":"): # check for colon is for testing try: file_datetime = datetime.datetime.fromtimestamp(os.path.getmtime(path)) except: file_datetime = None if file_datetime is not None: data_element["datetime_modified"] = Utility.get_datetime_item_from_datetime(file_datetime) return [data_element] return list()
def write_data(self, data, file_datetime): """ Write data to the ndata file specified by reference. :param data: the numpy array data to write :param file_datetime: the datetime for the file """ with self.__lock: assert data is not None absolute_file_path = self.__file_path #logging.debug("WRITE data file %s for %s", absolute_file_path, key) make_directory_if_needed(os.path.dirname(absolute_file_path)) properties = self.read_properties() if os.path.exists( absolute_file_path) else dict() write_zip(absolute_file_path, data, properties) # convert to utc time. tz_minutes = Utility.local_utcoffset_minutes(file_datetime) timestamp = calendar.timegm( file_datetime.timetuple()) - tz_minutes * 60 os.utime(absolute_file_path, (time.time(), timestamp))
def __write_properties_to_dataset(self, properties): with self.__lock: assert self.__dataset is not None class JSONEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, Geometry.IntPoint) or isinstance( obj, Geometry.IntSize) or isinstance( obj, Geometry.IntRect) or isinstance( obj, Geometry.FloatPoint) or isinstance( obj, Geometry.FloatSize) or isinstance( obj, Geometry.FloatRect): return tuple(obj) else: return json.JSONEncoder.default(self, obj) json_io = io.StringIO() json.dump(Utility.clean_dict(properties), json_io, cls=JSONEncoder) json_str = json_io.getvalue() self.__dataset.attrs["properties"] = json_str
def read_data_elements(self, extension: str, path: pathlib.Path) -> typing.List[DataElementType]: data = None try: data = read_image_from_file(path) except Exception as e: print(e) if data is not None: data_element: DataElementType = dict() data_element["version"] = 1 data_element["data"] = data if path.exists(): try: file_datetime = datetime.datetime.fromtimestamp( os.path.getmtime(path)) data_element[ "datetime_modified"] = Utility.get_datetime_item_from_datetime( file_datetime) except Exception: pass return [data_element] return list()
def read_library(persistent_storage_system, ignore_older_files) -> typing.Dict: """Read data items from the data reference handler and return as a list. Data items will have persistent_object_context set upon return, but caller will need to call finish_reading on each of the data items. """ data_item_uuids = set() utilized_deletions = set() # the uuid's skipped due to being deleted deletions = list() reader_info_list, library_updates = auto_migrate_storage_system( persistent_storage_system=persistent_storage_system, new_persistent_storage_system=persistent_storage_system, data_item_uuids=data_item_uuids, deletions=deletions, utilized_deletions=utilized_deletions, ignore_older_files=ignore_older_files) # next, for each auto migration, create a temporary storage system and read items from that storage system # using auto_migrate_storage_system. the data items returned will have been copied to the current storage # system (persistent object context). for auto_migration in reversed( persistent_storage_system.get_auto_migrations()): old_persistent_storage_system = FileStorageSystem( auto_migration.library_path, auto_migration.paths ) if auto_migration.paths else auto_migration.storage_system new_reader_info_list, new_library_updates = auto_migrate_storage_system( persistent_storage_system=old_persistent_storage_system, new_persistent_storage_system=persistent_storage_system, data_item_uuids=data_item_uuids, deletions=deletions, utilized_deletions=utilized_deletions, ignore_older_files=ignore_older_files) reader_info_list.extend(new_reader_info_list) library_updates.update(new_library_updates) assert len(reader_info_list) == len(data_item_uuids) library_storage_properties = persistent_storage_system.library_storage_properties for reader_info in reader_info_list: properties = reader_info.properties properties = Utility.clean_dict( copy.deepcopy(properties) if properties else dict()) version = properties.get("version", 0) if version == DataItem.DataItem.writer_version: data_item_uuid = uuid.UUID(properties.get("uuid", uuid.uuid4())) library_update = library_updates.get(data_item_uuid, dict()) library_storage_properties.setdefault( "connections", list()).extend(library_update.get("connections", list())) library_storage_properties.setdefault( "computations", list()).extend(library_update.get("computations", list())) library_storage_properties.setdefault( "display_items", list()).extend(library_update.get("display_items", list())) # mark deletions that need to be tracked because they've been deleted but are also present in older libraries # and would be migrated during reading unless they explicitly are prevented from doing so (via data_item_deletions). # utilized deletions are the ones that were attempted; if nothing was attempted, then no reason to track it anymore # since there is nothing to migrate in the future. library_storage_properties["data_item_deletions"] = [ str(uuid_) for uuid_ in utilized_deletions ] connections_list = library_storage_properties.get("connections", list()) assert len(connections_list) == len( {connection.get("uuid") for connection in connections_list}) computations_list = library_storage_properties.get("computations", list()) assert len(computations_list) == len( {computation.get("uuid") for computation in computations_list}) # migrations if library_storage_properties.get("version", 0) < 2: for data_group_properties in library_storage_properties.get( "data_groups", list()): data_group_properties.pop("data_groups") display_item_references = data_group_properties.setdefault( "display_item_references", list()) data_item_uuid_strs = data_group_properties.pop( "data_item_uuids", list()) for data_item_uuid_str in data_item_uuid_strs: for display_item_properties in library_storage_properties.get( "display_items", list()): data_item_references = [ d.get("data_item_reference", None) for d in display_item_properties.get( "display_data_channels", list()) ] if data_item_uuid_str in data_item_references: display_item_references.append( display_item_properties["uuid"]) data_item_uuid_to_display_item_uuid_map = dict() data_item_uuid_to_display_item_dict_map = dict() display_to_display_item_map = dict() display_to_display_data_channel_map = dict() for display_item_properties in library_storage_properties.get( "display_items", list()): display_to_display_item_map[display_item_properties["display"][ "uuid"]] = display_item_properties["uuid"] display_to_display_data_channel_map[display_item_properties[ "display"]["uuid"]] = display_item_properties[ "display_data_channels"][0]["uuid"] data_item_references = [ d.get("data_item_reference", None) for d in display_item_properties.get("display_data_channels", list()) ] for data_item_uuid_str in data_item_references: data_item_uuid_to_display_item_uuid_map.setdefault( data_item_uuid_str, display_item_properties["uuid"]) data_item_uuid_to_display_item_dict_map.setdefault( data_item_uuid_str, display_item_properties) display_item_properties.pop("display", None) for workspace_properties in library_storage_properties.get( "workspaces", list()): def replace1(d): if "children" in d: for dd in d["children"]: replace1(dd) if "data_item_uuid" in d: data_item_uuid_str = d.pop("data_item_uuid") display_item_uuid_str = data_item_uuid_to_display_item_uuid_map.get( data_item_uuid_str) if display_item_uuid_str: d["display_item_uuid"] = display_item_uuid_str replace1(workspace_properties["layout"]) for connection_dict in library_storage_properties.get( "connections", list()): source_uuid_str = connection_dict["source_uuid"] if connection_dict["type"] == "interval-list-connection": connection_dict[ "source_uuid"] = display_to_display_item_map.get( source_uuid_str, None) if connection_dict[ "type"] == "property-connection" and connection_dict[ "source_property"] == "slice_interval": connection_dict[ "source_uuid"] = display_to_display_data_channel_map.get( source_uuid_str, None) def fix_specifier(specifier_dict): if specifier_dict.get("type") in ("data_item", "display_xdata", "cropped_xdata", "cropped_display_xdata", "filter_xdata", "filtered_xdata"): if specifier_dict.get( "uuid") in data_item_uuid_to_display_item_dict_map: specifier_dict[ "uuid"] = data_item_uuid_to_display_item_dict_map[ specifier_dict["uuid"]]["display_data_channels"][ 0]["uuid"] else: specifier_dict.pop("uuid", None) if specifier_dict.get("type") == "data_item": specifier_dict["type"] = "data_source" if specifier_dict.get("type") == "data_item_object": specifier_dict["type"] = "data_item" if specifier_dict.get("type") == "region": specifier_dict["type"] = "graphic" for computation_dict in library_storage_properties.get( "computations", list()): for variable_dict in computation_dict.get("variables", list()): if "specifier" in variable_dict: specifier_dict = variable_dict["specifier"] if specifier_dict is not None: fix_specifier(specifier_dict) if "secondary_specifier" in variable_dict: specifier_dict = variable_dict["secondary_specifier"] if specifier_dict is not None: fix_specifier(specifier_dict) for result_dict in computation_dict.get("results", list()): fix_specifier(result_dict["specifier"]) library_storage_properties[ "version"] = DocumentModel.DocumentModel.library_version # TODO: add consistency checks: no duplicated items [by uuid] such as connections or computations or data items assert library_storage_properties[ "version"] == DocumentModel.DocumentModel.library_version persistent_storage_system.rewrite_properties(library_storage_properties) properties = copy.deepcopy(library_storage_properties) for reader_info in reader_info_list: data_item_properties = Utility.clean_dict( reader_info.properties if reader_info.properties else dict()) if data_item_properties.get("version", 0) == DataItem.DataItem.writer_version: data_item_properties["__large_format"] = reader_info.large_format data_item_properties["__identifier"] = reader_info.identifier properties.setdefault("data_items", list()).append(data_item_properties) def data_item_created(data_item_properties: typing.Mapping) -> str: return data_item_properties.get("created", "1900-01-01T00:00:00.000000") properties["data_items"] = sorted(properties.get("data_items", list()), key=data_item_created) return properties
def write_properties(self, properties, file_datetime): self.__properties[self.__uuid] = Utility.clean_dict(properties)
def __write_data_dict(self) -> None: if self.__file_path: with Utility.AtomicFileWriter(self.__file_path) as fp: json.dump(self.__data_dict, fp, skipkeys=True, indent=4)
def load_plug_ins(document_location: str, data_location: str, root_dir: typing.Optional[str]) -> None: """Load plug-ins.""" global extensions # a list of directories in which sub-directories PlugIns will be searched. subdirectories = [] # the default location is where the directory main packages are located. if root_dir: subdirectories.append(root_dir) # also search the default data location; create directory there if it doesn't exist to make it easier for user. # default data location will be application specific. if data_location is not None: subdirectories.append(data_location) # create directories here if they don't exist plugins_dir = os.path.abspath(os.path.join(data_location, "PlugIns")) if not os.path.exists(plugins_dir): logging.info("Creating plug-ins directory %s", plugins_dir) os.makedirs(plugins_dir) # search the Nion/Swift subdirectory of the default document location too, # but don't create directories here - avoid polluting user visible directories. if document_location is not None: subdirectories.append(os.path.join(document_location, "Nion", "Swift")) # do not create them in documents if they don't exist. this location is optional. # build a list of directories that will be loaded as plug-ins. PlugInDir = collections.namedtuple("PlugInDir", ["directory", "relative_path"]) plugin_dirs = list() # track directories that have already been searched. seen_plugin_dirs = list() # for each subdirectory, look in PlugIns for sub-directories that represent the plug-ins. for subdirectory in subdirectories: plugins_dir = os.path.abspath(os.path.join(subdirectory, "PlugIns")) if os.path.exists(plugins_dir) and not plugins_dir in seen_plugin_dirs: logging.info("Loading plug-ins from %s", plugins_dir) # add the PlugIns directory to the system import path. sys.path.append(plugins_dir) # now build a list of sub-directories representing plug-ins within plugins_dir. sorted_relative_paths = sorted([d for d in os.listdir(plugins_dir) if os.path.isdir(os.path.join(plugins_dir, d))]) plugin_dirs.extend([PlugInDir(plugins_dir, sorted_relative_path) for sorted_relative_path in sorted_relative_paths]) # mark plugins_dir as 'seen' to avoid search it twice. seen_plugin_dirs.append(plugins_dir) else: logging.info("NOT Loading plug-ins from %s (missing)", plugins_dir) version_map: PersistentDictType = dict() module_exists_map: typing.Dict[str, bool] = dict() plugin_adapters: typing.List[typing.Union[ModuleAdapter, PlugInAdapter]] = list() import nionswift_plugin for module_info in pkgutil.iter_modules(getattr(nionswift_plugin, "__path__")): plugin_adapters.append(ModuleAdapter(getattr(nionswift_plugin, "__name__"), module_info)) for directory, relative_path in plugin_dirs: plugin_adapters.append(PlugInAdapter(directory, relative_path)) progress = True while progress: progress = False plugin_adapters_copy = copy.deepcopy(plugin_adapters) plugin_adapters = list() for plugin_adapter in plugin_adapters_copy: manifest_path = plugin_adapter.manifest_path manifest = plugin_adapter.manifest if manifest: manifest_valid = True if not "name" in manifest: logging.info("Invalid manifest (missing 'name'): %s", manifest_path) manifest_valid = False if not "identifier" in manifest: logging.info("Invalid manifest (missing 'identifier'): %s", manifest_path) manifest_valid = False if "identifier" in manifest and not re.match("[_\-a-zA-Z][_\-a-zA-Z0-9.]*$", manifest["identifier"]): logging.info("Invalid manifest (invalid 'identifier': '%s'): %s", manifest["identifier"], manifest_path) manifest_valid = False if not "version" in manifest: logging.info("Invalid manifest (missing 'version'): %s", manifest_path) manifest_valid = False if "requires" in manifest and not isinstance(manifest["requires"], list): logging.info("Invalid manifest ('requires' not a list): %s", manifest_path) manifest_valid = False if not manifest_valid: continue for module in manifest.get("modules", list()): if module in module_exists_map: module_exists = module_exists_map.get(module) else: module_exists = importlib.util.find_spec(module) is not None module_exists_map[module] = module_exists if not module_exists: logging.info("Plug-in '" + plugin_adapter.module_name + "' NOT loaded (" + plugin_adapter.module_path + ").") logging.info("Cannot satisfy requirement (%s): %s", module, manifest_path) manifest_valid = False break for requirement in manifest.get("requires", list()): # TODO: see https://packaging.pypa.io/en/latest/ requirement_components = requirement.split() if len(requirement_components) != 3 or requirement_components[1] != "~=": logging.info("Invalid manifest (requirement '%s' invalid): %s", requirement, manifest_path) manifest_valid = False break identifier, operator, version_specifier = requirement_components[0], requirement_components[1], requirement_components[2] if identifier in version_map: if Utility.compare_versions("~" + version_specifier, version_map[identifier]) != 0: logging.info("Plug-in '" + plugin_adapter.module_name + "' NOT loaded (" + plugin_adapter.module_path + ").") logging.info("Cannot satisfy requirement (%s): %s", requirement, manifest_path) manifest_valid = False break else: # requirements not loaded yet; add back to plugin_adapters, but don't mark progress since nothing was loaded. logging.info("Plug-in '" + plugin_adapter.module_name + "' delayed (%s) (" + plugin_adapter.module_path + ").", requirement) plugin_adapters.append(plugin_adapter) manifest_valid = False break if not manifest_valid: continue version_map[manifest["identifier"]] = manifest["version"] # read the manifests, if any # repeat loop of plug-ins until no plug-ins left in the list # if all dependencies satisfied for a plug-in, load it # otherwise defer until next round # stop if no plug-ins loaded in the round # count on the user to have correct dependencies module = plugin_adapter.load() if module: __modules.append(module) progress = True for plugin_adapter in plugin_adapters: logging.info("Plug-in '" + plugin_adapter.module_name + "' NOT loaded (requirements) (" + plugin_adapter.module_path + ").") notify_modules("run")
def stringify_item(item): date_utc = item[1] tz_minutes = Utility.local_utcoffset_minutes(date_utc) date_local = date_utc + datetime.timedelta(minutes=tz_minutes) return str(os.path.basename(os.path.dirname(item[0]))) + " (" + date_local.strftime("%c") + ")"
def get_interactive(self, version: typing.Optional[str] = None) -> typing.Any: actual_version = "1.0.0" if Utility.compare_versions(version or str(), actual_version) > 0: raise NotImplementedError("API requested version %s is greater than %s." % (version, actual_version)) return interactive_session
def convert_data_element_to_data_and_metadata_1(data_element) -> DataAndMetadata.DataAndMetadata: """Convert a data element to xdata. No data copying occurs. The data element can have the following keys: data (required) is_sequence, collection_dimension_count, datum_dimension_count (optional description of the data) spatial_calibrations (optional list of spatial calibration dicts, scale, offset, units) intensity_calibration (optional intensity calibration dict, scale, offset, units) metadata (optional) properties (get stored into metadata.hardware_source) one of either timestamp or datetime_modified if datetime_modified (dst, tz) it is converted and used as timestamp then timezone gets stored into metadata.description.timezone. """ # data. takes ownership. data = data_element["data"] dimensional_shape = Image.dimensional_shape_from_data(data) is_sequence = data_element.get("is_sequence", False) dimension_count = len(Image.dimensional_shape_from_data(data)) adjusted_dimension_count = dimension_count - (1 if is_sequence else 0) collection_dimension_count = data_element.get("collection_dimension_count", 2 if adjusted_dimension_count in (3, 4) else 0) datum_dimension_count = data_element.get("datum_dimension_count", adjusted_dimension_count - collection_dimension_count) data_descriptor = DataAndMetadata.DataDescriptor(is_sequence, collection_dimension_count, datum_dimension_count) # dimensional calibrations dimensional_calibrations = None if "spatial_calibrations" in data_element: dimensional_calibrations_list = data_element.get("spatial_calibrations") if len(dimensional_calibrations_list) == len(dimensional_shape): dimensional_calibrations = list() for dimension_calibration in dimensional_calibrations_list: offset = float(dimension_calibration.get("offset", 0.0)) scale = float(dimension_calibration.get("scale", 1.0)) units = dimension_calibration.get("units", "") units = str(units) if units is not None else str() if scale != 0.0: dimensional_calibrations.append(Calibration.Calibration(offset, scale, units)) else: dimensional_calibrations.append(Calibration.Calibration()) # intensity calibration intensity_calibration = None if "intensity_calibration" in data_element: intensity_calibration_dict = data_element.get("intensity_calibration") offset = float(intensity_calibration_dict.get("offset", 0.0)) scale = float(intensity_calibration_dict.get("scale", 1.0)) units = intensity_calibration_dict.get("units", "") units = str(units) if units is not None else str() if scale != 0.0: intensity_calibration = Calibration.Calibration(offset, scale, units) # properties (general tags) metadata = dict() if "metadata" in data_element: metadata.update(Utility.clean_dict(data_element.get("metadata"))) if "properties" in data_element and data_element["properties"]: hardware_source_metadata = metadata.setdefault("hardware_source", dict()) hardware_source_metadata.update(Utility.clean_dict(data_element.get("properties"))) # dates are _local_ time and must use this specific ISO 8601 format. 2013-11-17T08:43:21.389391 # time zones are offsets (east of UTC) in the following format "+HHMM" or "-HHMM" # daylight savings times are time offset (east of UTC) in format "+MM" or "-MM" # timezone is for conversion and is the Olson timezone string. # datetime.datetime.strptime(datetime.datetime.isoformat(datetime.datetime.now()), "%Y-%m-%dT%H:%M:%S.%f" ) # datetime_modified, datetime_modified_tz, datetime_modified_dst, datetime_modified_tzname is the time at which this image was modified. # datetime_original, datetime_original_tz, datetime_original_dst, datetime_original_tzname is the time at which this image was created. timestamp = data_element.get("timestamp", datetime.datetime.utcnow()) datetime_item = data_element.get("datetime_modified", Utility.get_datetime_item_from_utc_datetime(timestamp)) local_datetime = Utility.get_datetime_from_datetime_item(datetime_item) dst_value = datetime_item.get("dst", "+00") tz_value = datetime_item.get("tz", "+0000") timezone = datetime_item.get("timezone") time_zone = { "dst": dst_value, "tz": tz_value} if timezone is not None: time_zone["timezone"] = timezone # note: dst is informational only; tz already include dst tz_adjust = (int(tz_value[1:3]) * 60 + int(tz_value[3:5])) * (-1 if tz_value[0] == '-' else 1) utc_datetime = local_datetime - datetime.timedelta(minutes=tz_adjust) # tz_adjust already contains dst_adjust timestamp = utc_datetime return DataAndMetadata.new_data_and_metadata(data, intensity_calibration=intensity_calibration, dimensional_calibrations=dimensional_calibrations, metadata=metadata, timestamp=timestamp, data_descriptor=data_descriptor, timezone=timezone, timezone_offset=tz_value)