class ROCrate(Resource): id = db.Column(db.Integer, db.ForeignKey(Resource.id), primary_key=True) hosting_service_id = db.Column(db.Integer, db.ForeignKey("resource.id"), nullable=True) hosting_service = db.relationship("Resource", uselist=False, backref=db.backref("ro_crates", cascade="all, delete-orphan"), foreign_keys=[hosting_service_id]) _metadata = db.Column("metadata", JSON, nullable=True) _local_path = None _metadata_loaded = False __roc_helper = None __mapper_args__ = { 'polymorphic_identity': 'ro_crate', "inherit_condition": id == Resource.id } def __init__(self, uri, uuid=None, name=None, version=None, hosting_service=None) -> None: super().__init__(uri, uuid=uuid, name=name, version=version) self.hosting_service = hosting_service self.__roc_helper = None @hybrid_property def crate_metadata(self): if not self._metadata_loaded: self.load_metadata() return self._metadata @hybrid_property def roc_suites(self): return get_roc_suites(self._roc_helper) def get_roc_suite(self, roc_suite_identifier): try: return self.roc_suites[roc_suite_identifier] except KeyError: logger.warning("Unable to find the roc_suite with identifier: %r", roc_suite_identifier) return None @property def dataset_name(self): return self._roc_helper.name @property def _roc_helper(self): if not self.__roc_helper: if not self._metadata_loaded: self.load_metadata() if not self.__roc_helper: raise RuntimeError("ROCrate not correctly loaded") return self.__roc_helper def _get_authorizations(self): authorizations = self.authorizations.copy() authorizations.append(None) return authorizations def load_metadata(self) -> dict: errors = [] # try either with authorization header and without authorization for authorization in self._get_authorizations(): try: auth_header = authorization.as_http_header() if authorization else None logger.debug(auth_header) self.__roc_helper, self._metadata = \ self.load_metadata_files(self.uri, authorization_header=auth_header) self._metadata_loaded = True return self._metadata except lm_exceptions.NotAuthorizedException as e: logger.info("Caught authorization error exception while downloading and processing RO-crate: %s", e) errors.append(str(e)) raise lm_exceptions.NotAuthorizedException(detail=f"Not authorized to download {self.uri}", original_errors=errors) @classmethod def load_metadata_files(cls, roc_link, authorization_header=None): with tempfile.TemporaryDirectory() as tmpdir: tmpdir_path = Path(tmpdir) local_zip = download_url(roc_link, target_path=(tmpdir_path / 'rocrate.zip').as_posix(), authorization=authorization_header) logger.debug("ZIP Archive: %s", local_zip) extracted_roc_path = tmpdir_path / 'extracted-rocrate' logger.info("Extracting RO Crate to %s", extracted_roc_path) extract_zip(local_zip, target_path=extracted_roc_path.as_posix()) if logger.isEnabledFor(logging.DEBUG): logger.debug("RO-crate contents: %s", ', '.join(str(s) for s in extracted_roc_path.iterdir())) try: crate = ROCrateHelper(extracted_roc_path.as_posix()) metadata_path = extracted_roc_path / crate.metadata.id with open(metadata_path, "rt") as f: metadata = json.load(f) return crate, metadata except Exception as e: raise lm_exceptions.NotValidROCrateException(detail=str(e))
class WorkflowVersion(ROCrate): id = db.Column(db.Integer, db.ForeignKey(ROCrate.id), primary_key=True) submitter_id = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True) workflow_id = \ db.Column(db.Integer, db.ForeignKey("workflow.id"), nullable=False) workflow = db.relationship( "Workflow", foreign_keys=[workflow_id], cascade="all", backref=db.backref( "versions", cascade="all, delete-orphan", collection_class=attribute_mapped_collection('version'))) test_suites = db.relationship("TestSuite", back_populates="workflow_version", cascade="all, delete") submitter = db.relationship( "User", uselist=False, backref=db.backref("workflows", cascade="all, delete-orphan", collection_class=WorkflowVersionCollection)) roc_link = association_proxy('ro_crate', 'uri') __mapper_args__ = {'polymorphic_identity': 'workflow_version'} def __init__(self, workflow: Workflow, uri, version, submitter: User, uuid=None, name=None, hosting_service: models.WorkflowRegistry = None) -> None: super().__init__(uri, uuid=uuid, name=name, version=version, hosting_service=hosting_service) self.submitter = submitter self.workflow = workflow def __repr__(self): return '<WorkflowVersion ({}, {}), name: {}, ro_crate link {}>'.format( self.uuid, self.version, self.name, self.roc_link) def check_health(self) -> dict: health = {'healthy': True, 'issues': []} for suite in self.test_suites: for test_instance in suite.test_instances: try: testing_service = test_instance.testing_service if not testing_service.last_test_build.is_successful(): health["healthy"] = False except lm_exceptions.TestingServiceException as e: health["issues"].append(str(e)) health["healthy"] = "Unknown" return health @hybrid_property def authorizations(self): auths = [a for a in self._authorizations] if self.hosting_service and self.submitter: for auth in self.submitter.get_authorization(self.hosting_service): auths.append(auth) return auths @hybrid_property def workflow_registry(self) -> models.WorkflowRegistry: return self.hosting_service @hybrid_property def roc_link(self) -> str: return self.uri @property def is_latest(self) -> bool: return self.workflow.latest_version.version == self.version @property def previous_versions(self) -> List[str]: return [ w.version for w in self.workflow.versions.values() if w != self and w.version < self.version ] @property def previous_workflow_versions(self) -> List[models.WorkflowVersion]: return [ w for w in self.workflow.versions.values() if w != self and w.version < self.version ] @property def status(self) -> models.WorkflowStatus: return models.WorkflowStatus(self) @property def is_healthy(self) -> Union[bool, str]: return self.check_health()["healthy"] def add_test_suite(self, submitter: User, name: str = None, roc_suite: str = None, definition: object = None): return models.TestSuite(self, submitter, name=name, roc_suite=roc_suite, definition=definition) @property def submitter_identity(self): # Return the submitter identity wrt the registry identity = OAuthIdentity.find_by_user_id(self.submitter.id, self.workflow_registry.name) return identity.provider_user_id def to_dict(self, test_suite=False, test_build=False, test_output=False): health = self.check_health() data = { 'uuid': str(self.uuid), 'version': self.version, 'name': self.name, 'roc_link': self.roc_link, 'isHealthy': health["healthy"], 'issues': health["issues"] } if test_suite: data['test_suite'] = [ s.to_dict(test_build=test_build, test_output=test_output) for s in self.test_suites ] return data def delete(self): if len(self.workflow.versions) > 1: workflow = self.workflow self.workflow.remove_version(self) workflow.save() else: self.workflow.delete() @classmethod def all(cls) -> List[WorkflowVersion]: return cls.query.all() @classmethod def get_submitter_versions(cls, submitter: User) -> List[WorkflowVersion]: return cls.query.filter( WorkflowVersion.submitter_id == submitter.id).all() @classmethod def get_user_workflow_version(cls, owner: User, uuid, version) -> WorkflowVersion: try: return cls.query\ .join(Workflow, Workflow.id == cls.workflow_id)\ .join(Permission, Permission.resource_id == cls.id)\ .filter(Workflow.uuid == lm_utils.uuid_param(uuid))\ .filter(Permission.user_id == owner.id)\ .filter(cls.version == version).one() except NoResultFound as e: logger.exception(e) return None except Exception as e: raise lm_exceptions.LifeMonitorException(detail=str(e), stack=str(e)) @classmethod def get_user_workflow_versions(cls, owner: User) -> List[WorkflowVersion]: return cls.query\ .join(Permission)\ .filter(Permission.resource_id == cls.id, Permission.user_id == owner.id).all() @classmethod def get_hosted_workflow_version(cls, hosting_service: Resource, uuid, version) -> List[WorkflowVersion]: # TODO: replace WorkflowRegistry with a more general Entity try: return cls.query\ .join(WorkflowRegistry, cls.hosting_service)\ .join(Workflow, Workflow.id == cls.workflow_id)\ .filter(WorkflowRegistry.uuid == lm_utils.uuid_param(hosting_service.uuid))\ .filter(Workflow.uuid == lm_utils.uuid_param(uuid))\ .filter(cls.version == version)\ .order_by(WorkflowVersion.version.desc()).one() except NoResultFound as e: logger.debug(e) return None except Exception as e: raise lm_exceptions.LifeMonitorException(detail=str(e), stack=str(e)) @classmethod def get_hosted_workflow_versions( cls, hosting_service: Resource) -> List[WorkflowVersion]: # TODO: replace WorkflowRegistry with a more general Entity return cls.query\ .join(WorkflowRegistry, cls.hosting_service)\ .filter(WorkflowRegistry.uuid == lm_utils.uuid_param(hosting_service.uuid))\ .order_by(WorkflowVersion.version.desc()).all()
class TestInstance(db.Model, ModelMixin): uuid = db.Column(UUID, primary_key=True, default=_uuid.uuid4) type = db.Column(db.String(20), nullable=False) _test_suite_uuid = \ db.Column("test_suite_uuid", UUID, db.ForeignKey(TestSuite.uuid), nullable=False) name = db.Column(db.String, nullable=False) roc_instance = db.Column(db.String, nullable=True) resource = db.Column(db.Text, nullable=False) parameters = db.Column(JSON, nullable=True) submitter_id = db.Column(db.Integer, db.ForeignKey(models.User.id), nullable=True) # configure relationships submitter = db.relationship("User", uselist=False) test_suite = db.relationship("TestSuite", back_populates="test_instances", foreign_keys=[_test_suite_uuid]) testing_service_id = db.Column(UUID, db.ForeignKey("testing_service.uuid"), nullable=False) testing_service = db.relationship("TestingService", foreign_keys=[testing_service_id], backref=db.backref("test_instances", cascade="all, delete-orphan"), uselist=False) __mapper_args__ = { 'polymorphic_on': type, 'polymorphic_identity': 'unmanaged' } def __init__(self, testing_suite: TestSuite, submitter: models.User, test_name, test_resource, testing_service: models.TestingService, roc_instance: str = None) -> None: self.test_suite = testing_suite self.submitter = submitter self.name = test_name self.roc_instance = roc_instance self.resource = test_resource self.testing_service = testing_service def __repr__(self): return '<TestInstance {} on TestSuite {}>'.format(self.uuid, self.test_suite.uuid) @property def is_roc_instance(self): return self.roc_instance is not None @property def managed(self): return self.type != 'unmanaged' @property def last_test_build(self): return self.testing_service.get_last_test_build(self) def get_test_builds(self, limit=10): return self.testing_service.get_test_builds(self, limit=limit) def get_test_build(self, build_number): return self.testing_service.get_test_build(self, build_number) def to_dict(self, test_build=False, test_output=False): data = { 'uuid': str(self.uuid), 'name': self.name, 'parameters': self.parameters, 'testing_service': self.testing_service.to_dict(test_builds=False) } if test_build: data.update(self.testing_service.get_test_builds_as_dict(test_output=test_output)) return data @classmethod def all(cls) -> List[TestInstance]: return cls.query.all() @classmethod def find_by_uuid(cls, uuid) -> TestInstance: return cls.query.get(uuid)
class ROCrate(Resource): id = db.Column(db.Integer, db.ForeignKey(Resource.id), primary_key=True) hosting_service_id = db.Column(db.Integer, db.ForeignKey("resource.id"), nullable=True) hosting_service = db.relationship("Resource", uselist=False, backref=db.backref( "ro_crates", cascade="all, delete-orphan"), foreign_keys=[hosting_service_id]) _metadata = db.Column("metadata", JSON, nullable=True) _test_metadata = None _local_path = None _metadata_loaded = False __mapper_args__ = { 'polymorphic_identity': 'ro_crate', "inherit_condition": id == Resource.id } def __init__(self, uri, uuid=None, name=None, version=None, hosting_service=None) -> None: super().__init__(uri, uuid=uuid, name=name, version=version) self.hosting_service = hosting_service self._crate_helper = None @hybrid_property def crate_metadata(self): if not self._metadata_loaded: self.load_metadata() return self._metadata @property def dataset_name(self): if not self._metadata_loaded: self.load_metadata() if not self._crate_helper: raise RuntimeError("ROCrate not correctly loaded") return self._crate_helper.name @property def test_metadata(self): if not self._metadata_loaded: self.load_metadata() return self._test_metadata def _get_authorizations(self): authorizations = self.authorizations.copy() authorizations.append(None) return authorizations def load_metadata(self): errors = [] # try either with authorization hedaer and without authorization for authorization in self._get_authorizations(): try: auth_header = authorization.as_http_header( ) if authorization else None logger.debug(auth_header) self._crate_helper, self._metadata, self._test_metadata = \ self.load_metadata_files(self.uri, authorization_header=auth_header) self._metadata_loaded = True return self._metadata, self._test_metadata except Exception as e: errors.append(e) if len([ e for e in errors if isinstance(e, lm_exceptions.NotAuthorizedException) ]) == len(errors): raise lm_exceptions.NotAuthorizedException() raise lm_exceptions.LifeMonitorException("ROCrate download error", errors=errors) @staticmethod def extract_rocrate(roc_link, target_path=None, authorization_header=None): with tempfile.NamedTemporaryFile(dir="/tmp") as archive_path: zip_archive = download_url(roc_link, target_path=archive_path.name, authorization=authorization_header) logger.debug("ZIP Archive: %s", zip_archive) roc_path = target_path or Path(tempfile.mkdtemp(dir="/tmp")) logger.info("Extracting RO Crate @ %s", roc_path) extract_zip(archive_path, target_path=roc_path.as_posix()) return roc_path @classmethod def load_metadata_files(cls, roc_link, authorization_header=None): roc_path = cls.extract_rocrate( roc_link, authorization_header=authorization_header) try: roc_posix_path = roc_path.as_posix() logger.debug(os.listdir(roc_posix_path)) crate = ROCrateHelper(roc_posix_path) metadata_path = Path(roc_posix_path) / crate.metadata.id with open(metadata_path, "rt") as f: metadata = json.load(f) # create a new Workflow instance with the loaded metadata test_metadata = get_old_format_tests(crate) return crate, metadata, test_metadata finally: shutil.rmtree(roc_path, ignore_errors=True)