Example #1
0
class ROCrate(Resource):

    id = db.Column(db.Integer, db.ForeignKey(Resource.id), primary_key=True)
    hosting_service_id = db.Column(db.Integer, db.ForeignKey("resource.id"), nullable=True)
    hosting_service = db.relationship("Resource", uselist=False,
                                      backref=db.backref("ro_crates", cascade="all, delete-orphan"),
                                      foreign_keys=[hosting_service_id])
    _metadata = db.Column("metadata", JSON, nullable=True)
    _local_path = None
    _metadata_loaded = False
    __roc_helper = None

    __mapper_args__ = {
        'polymorphic_identity': 'ro_crate',
        "inherit_condition": id == Resource.id
    }

    def __init__(self, uri, uuid=None, name=None,
                 version=None, hosting_service=None) -> None:
        super().__init__(uri, uuid=uuid, name=name, version=version)
        self.hosting_service = hosting_service
        self.__roc_helper = None

    @hybrid_property
    def crate_metadata(self):
        if not self._metadata_loaded:
            self.load_metadata()
        return self._metadata

    @hybrid_property
    def roc_suites(self):
        return get_roc_suites(self._roc_helper)

    def get_roc_suite(self, roc_suite_identifier):
        try:
            return self.roc_suites[roc_suite_identifier]
        except KeyError:
            logger.warning("Unable to find the roc_suite with identifier: %r", roc_suite_identifier)
        return None

    @property
    def dataset_name(self):
        return self._roc_helper.name

    @property
    def _roc_helper(self):
        if not self.__roc_helper:
            if not self._metadata_loaded:
                self.load_metadata()
        if not self.__roc_helper:
            raise RuntimeError("ROCrate not correctly loaded")
        return self.__roc_helper

    def _get_authorizations(self):
        authorizations = self.authorizations.copy()
        authorizations.append(None)
        return authorizations

    def load_metadata(self) -> dict:
        errors = []
        # try either with authorization header and without authorization
        for authorization in self._get_authorizations():
            try:
                auth_header = authorization.as_http_header() if authorization else None
                logger.debug(auth_header)
                self.__roc_helper, self._metadata = \
                    self.load_metadata_files(self.uri, authorization_header=auth_header)
                self._metadata_loaded = True
                return self._metadata
            except lm_exceptions.NotAuthorizedException as e:
                logger.info("Caught authorization error exception while downloading and processing RO-crate: %s", e)
                errors.append(str(e))
        raise lm_exceptions.NotAuthorizedException(detail=f"Not authorized to download {self.uri}", original_errors=errors)

    @classmethod
    def load_metadata_files(cls, roc_link, authorization_header=None):
        with tempfile.TemporaryDirectory() as tmpdir:
            tmpdir_path = Path(tmpdir)
            local_zip = download_url(roc_link,
                                     target_path=(tmpdir_path / 'rocrate.zip').as_posix(),
                                     authorization=authorization_header)

            logger.debug("ZIP Archive: %s", local_zip)
            extracted_roc_path = tmpdir_path / 'extracted-rocrate'
            logger.info("Extracting RO Crate to %s", extracted_roc_path)
            extract_zip(local_zip, target_path=extracted_roc_path.as_posix())

            if logger.isEnabledFor(logging.DEBUG):
                logger.debug("RO-crate contents: %s", ', '.join(str(s) for s in extracted_roc_path.iterdir()))

            try:
                crate = ROCrateHelper(extracted_roc_path.as_posix())
                metadata_path = extracted_roc_path / crate.metadata.id
                with open(metadata_path, "rt") as f:
                    metadata = json.load(f)
                return crate, metadata
            except Exception as e:
                raise lm_exceptions.NotValidROCrateException(detail=str(e))
Example #2
0
class TestSuite(db.Model, ModelMixin):
    uuid = db.Column(UUID, primary_key=True, default=_uuid.uuid4)
    _workflow_version_id = db.Column("workflow_version_id",
                                     db.Integer,
                                     db.ForeignKey(
                                         models.workflows.WorkflowVersion.id),
                                     nullable=False)
    workflow_version = db.relationship("WorkflowVersion",
                                       back_populates="test_suites")
    test_definition = db.Column(JSON, nullable=False)
    submitter_id = db.Column(db.Integer,
                             db.ForeignKey(User.id),
                             nullable=False)
    submitter = db.relationship("User", uselist=False)
    test_instances = db.relationship("TestInstance",
                                     back_populates="test_suite",
                                     cascade="all, delete")

    def __init__(self, w: models.workflows.WorkflowVersion, submitter: User,
                 test_definition: object) -> None:
        self.workflow_version = w
        self.submitter = submitter
        self.test_definition = test_definition
        self._parse_test_definition()

    def __repr__(self):
        return '<TestSuite {} of workflow {} (version {})>'.format(
            self.uuid, self.workflow_version.uuid,
            self.workflow_version.version)

    def _parse_test_definition(self):
        try:
            for test in self.test_definition["test"]:
                test = tm.Test.from_json(test)
                for instance in test.instance:
                    logger.debug("Instance: %r", instance)
                    testing_service = models.TestingService.get_instance(
                        instance.service.type, instance.service.url)
                    assert testing_service, "Testing service not initialized"
                    logger.debug("Created TestService: %r", testing_service)
                    test_instance = models.TestInstance(
                        self, self.submitter, test.name,
                        instance.service.resource, testing_service)
                    logger.debug("Created TestInstance: %r", test_instance)
        except KeyError as e:
            raise lm_exceptions.SpecificationNotValidException(
                f"Missing property: {e}")

    @property
    def status(self) -> models.SuiteStatus:
        return models.SuiteStatus(self)

    def get_test_instance_by_name(self, name) -> list:
        result = []
        for ti in self.test_instances:
            if ti.name == name:
                result.append(ti)
        return result

    def to_dict(self, test_build=False, test_output=False) -> dict:
        return {
            'uuid':
            str(self.uuid),
            'test': [
                t.to_dict(test_build=test_build, test_output=test_output)
                for t in self.test_instances
            ]
        }

    def add_test_instance(self, submitter: User, test_name,
                          testing_service_type, testing_service_url,
                          testing_service_resource):
        testing_service = \
            models.TestingService.get_instance(testing_service_type, testing_service_url)
        test_instance = models.TestInstance(self, submitter, test_name,
                                            testing_service_resource,
                                            testing_service)
        logger.debug("Created TestInstance: %r", test_instance)
        return test_instance

    @property
    def tests(self) -> Optional[dict]:
        if not self.test_definition:
            raise lm_exceptions.SpecificationNotDefinedException(
                'Not test definition for the test suite {}'.format(self.uuid))
        if "test" not in self.test_definition:
            raise lm_exceptions.SpecificationNotValidException(
                "'test' property not found")
        # TODO: implement a caching mechanism: with a custom setter for the test_definition collection
        result = {}
        for test in self.test_definition["test"]:
            result[test["name"]] = Test(
                self, test["name"],
                test["specification"] if "specification" in test else None)
        return result

    @classmethod
    def all(cls) -> List[TestSuite]:
        return cls.query.all()

    @classmethod
    def find_by_uuid(cls, uuid) -> TestSuite:
        return cls.query.get(uuid)
Example #3
0
class TestInstance(db.Model, ModelMixin):
    uuid = db.Column(UUID, primary_key=True, default=_uuid.uuid4)
    type = db.Column(db.String(20), nullable=False)
    _test_suite_uuid = \
        db.Column("test_suite_uuid", UUID, db.ForeignKey(TestSuite.uuid), nullable=False)
    name = db.Column(db.String, nullable=False)
    roc_instance = db.Column(db.String, nullable=True)
    resource = db.Column(db.Text, nullable=False)
    parameters = db.Column(JSON, nullable=True)
    submitter_id = db.Column(db.Integer,
                             db.ForeignKey(models.User.id), nullable=True)
    # configure relationships
    submitter = db.relationship("User", uselist=False)
    test_suite = db.relationship("TestSuite",
                                 back_populates="test_instances",
                                 foreign_keys=[_test_suite_uuid])
    testing_service_id = db.Column(UUID, db.ForeignKey("testing_service.uuid"), nullable=False)
    testing_service = db.relationship("TestingService",
                                      foreign_keys=[testing_service_id],
                                      backref=db.backref("test_instances", cascade="all, delete-orphan"),
                                      uselist=False)

    __mapper_args__ = {
        'polymorphic_on': type,
        'polymorphic_identity': 'unmanaged'
    }

    def __init__(self, testing_suite: TestSuite, submitter: models.User,
                 test_name, test_resource, testing_service: models.TestingService,
                 roc_instance: str = None) -> None:
        self.test_suite = testing_suite
        self.submitter = submitter
        self.name = test_name
        self.roc_instance = roc_instance
        self.resource = test_resource
        self.testing_service = testing_service

    def __repr__(self):
        return '<TestInstance {} on TestSuite {}>'.format(self.uuid, self.test_suite.uuid)

    @property
    def is_roc_instance(self):
        return self.roc_instance is not None

    @property
    def managed(self):
        return self.type != 'unmanaged'

    @property
    def last_test_build(self):
        return self.testing_service.get_last_test_build(self)

    def get_test_builds(self, limit=10):
        return self.testing_service.get_test_builds(self, limit=limit)

    def get_test_build(self, build_number):
        return self.testing_service.get_test_build(self, build_number)

    def to_dict(self, test_build=False, test_output=False):
        data = {
            'uuid': str(self.uuid),
            'name': self.name,
            'parameters': self.parameters,
            'testing_service': self.testing_service.to_dict(test_builds=False)
        }
        if test_build:
            data.update(self.testing_service.get_test_builds_as_dict(test_output=test_output))
        return data

    @classmethod
    def all(cls) -> List[TestInstance]:
        return cls.query.all()

    @classmethod
    def find_by_uuid(cls, uuid) -> TestInstance:
        return cls.query.get(uuid)
Example #4
0
class WorkflowVersion(ROCrate):
    id = db.Column(db.Integer, db.ForeignKey(ROCrate.id), primary_key=True)
    submitter_id = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
    workflow_id = \
        db.Column(db.Integer, db.ForeignKey("workflow.id"), nullable=False)
    workflow = db.relationship(
        "Workflow",
        foreign_keys=[workflow_id],
        cascade="all",
        backref=db.backref(
            "versions",
            cascade="all, delete-orphan",
            collection_class=attribute_mapped_collection('version')))
    test_suites = db.relationship("TestSuite",
                                  back_populates="workflow_version",
                                  cascade="all, delete")
    submitter = db.relationship(
        "User",
        uselist=False,
        backref=db.backref("workflows",
                           cascade="all, delete-orphan",
                           collection_class=WorkflowVersionCollection))
    roc_link = association_proxy('ro_crate', 'uri')

    __mapper_args__ = {'polymorphic_identity': 'workflow_version'}

    def __init__(self,
                 workflow: Workflow,
                 uri,
                 version,
                 submitter: User,
                 uuid=None,
                 name=None,
                 hosting_service: models.WorkflowRegistry = None) -> None:
        super().__init__(uri,
                         uuid=uuid,
                         name=name,
                         version=version,
                         hosting_service=hosting_service)
        self.submitter = submitter
        self.workflow = workflow

    def __repr__(self):
        return '<WorkflowVersion ({}, {}), name: {}, ro_crate link {}>'.format(
            self.uuid, self.version, self.name, self.roc_link)

    def check_health(self) -> dict:
        health = {'healthy': True, 'issues': []}
        for suite in self.test_suites:
            for test_instance in suite.test_instances:
                try:
                    testing_service = test_instance.testing_service
                    if not testing_service.last_test_build.is_successful():
                        health["healthy"] = False
                except lm_exceptions.TestingServiceException as e:
                    health["issues"].append(str(e))
                    health["healthy"] = "Unknown"
        return health

    @hybrid_property
    def authorizations(self):
        auths = [a for a in self._authorizations]
        if self.hosting_service and self.submitter:
            for auth in self.submitter.get_authorization(self.hosting_service):
                auths.append(auth)
        return auths

    @hybrid_property
    def workflow_registry(self) -> models.WorkflowRegistry:
        return self.hosting_service

    @hybrid_property
    def roc_link(self) -> str:
        return self.uri

    @property
    def is_latest(self) -> bool:
        return self.workflow.latest_version.version == self.version

    @property
    def previous_versions(self) -> List[str]:
        return [
            w.version for w in self.workflow.versions.values()
            if w != self and w.version < self.version
        ]

    @property
    def previous_workflow_versions(self) -> List[models.WorkflowVersion]:
        return [
            w for w in self.workflow.versions.values()
            if w != self and w.version < self.version
        ]

    @property
    def status(self) -> models.WorkflowStatus:
        return models.WorkflowStatus(self)

    @property
    def is_healthy(self) -> Union[bool, str]:
        return self.check_health()["healthy"]

    def add_test_suite(self,
                       submitter: User,
                       name: str = None,
                       roc_suite: str = None,
                       definition: object = None):
        return models.TestSuite(self,
                                submitter,
                                name=name,
                                roc_suite=roc_suite,
                                definition=definition)

    @property
    def submitter_identity(self):
        # Return the submitter identity wrt the registry
        identity = OAuthIdentity.find_by_user_id(self.submitter.id,
                                                 self.workflow_registry.name)
        return identity.provider_user_id

    def to_dict(self, test_suite=False, test_build=False, test_output=False):
        health = self.check_health()
        data = {
            'uuid': str(self.uuid),
            'version': self.version,
            'name': self.name,
            'roc_link': self.roc_link,
            'isHealthy': health["healthy"],
            'issues': health["issues"]
        }
        if test_suite:
            data['test_suite'] = [
                s.to_dict(test_build=test_build, test_output=test_output)
                for s in self.test_suites
            ]
        return data

    def delete(self):
        if len(self.workflow.versions) > 1:
            workflow = self.workflow
            self.workflow.remove_version(self)
            workflow.save()
        else:
            self.workflow.delete()

    @classmethod
    def all(cls) -> List[WorkflowVersion]:
        return cls.query.all()

    @classmethod
    def get_submitter_versions(cls, submitter: User) -> List[WorkflowVersion]:
        return cls.query.filter(
            WorkflowVersion.submitter_id == submitter.id).all()

    @classmethod
    def get_user_workflow_version(cls, owner: User, uuid,
                                  version) -> WorkflowVersion:
        try:
            return cls.query\
                .join(Workflow, Workflow.id == cls.workflow_id)\
                .join(Permission, Permission.resource_id == cls.id)\
                .filter(Workflow.uuid == lm_utils.uuid_param(uuid))\
                .filter(Permission.user_id == owner.id)\
                .filter(cls.version == version).one()
        except NoResultFound as e:
            logger.exception(e)
            return None
        except Exception as e:
            raise lm_exceptions.LifeMonitorException(detail=str(e),
                                                     stack=str(e))

    @classmethod
    def get_user_workflow_versions(cls, owner: User) -> List[WorkflowVersion]:
        return cls.query\
            .join(Permission)\
            .filter(Permission.resource_id == cls.id, Permission.user_id == owner.id).all()

    @classmethod
    def get_hosted_workflow_version(cls, hosting_service: Resource, uuid,
                                    version) -> List[WorkflowVersion]:
        # TODO: replace WorkflowRegistry with a more general Entity
        try:
            return cls.query\
                .join(WorkflowRegistry, cls.hosting_service)\
                .join(Workflow, Workflow.id == cls.workflow_id)\
                .filter(WorkflowRegistry.uuid == lm_utils.uuid_param(hosting_service.uuid))\
                .filter(Workflow.uuid == lm_utils.uuid_param(uuid))\
                .filter(cls.version == version)\
                .order_by(WorkflowVersion.version.desc()).one()
        except NoResultFound as e:
            logger.debug(e)
            return None
        except Exception as e:
            raise lm_exceptions.LifeMonitorException(detail=str(e),
                                                     stack=str(e))

    @classmethod
    def get_hosted_workflow_versions(
            cls, hosting_service: Resource) -> List[WorkflowVersion]:
        # TODO: replace WorkflowRegistry with a more general Entity
        return cls.query\
            .join(WorkflowRegistry, cls.hosting_service)\
            .filter(WorkflowRegistry.uuid == lm_utils.uuid_param(hosting_service.uuid))\
            .order_by(WorkflowVersion.version.desc()).all()
Example #5
0
class Workflow(Resource):
    id = db.Column(db.Integer, db.ForeignKey(Resource.id), primary_key=True)

    external_ns = "external-id:"

    __mapper_args__ = {'polymorphic_identity': 'workflow'}

    def __init__(self,
                 uri=None,
                 uuid=None,
                 identifier=None,
                 version=None,
                 name=None) -> None:
        super().__init__(uri=uri or f"{self.external_ns}",
                         uuid=uuid,
                         version=version,
                         name=name)
        if identifier is not None:
            self.external_id = identifier

    def __repr__(self):
        return '<Workflow ({}), name: {}>'.format(self.uuid, self.name)

    @hybrid_property
    def external_id(self):
        r = self.uri.replace(self.external_ns, "")
        return r if len(r) > 0 else None

    @external_id.setter
    def external_id(self, value):
        self.uri = f"{self.external_ns}{value}"

    @hybrid_property
    def latest_version(self) -> WorkflowVersion:
        return max(self.versions.values(), key=lambda v: v.version)

    def add_version(self,
                    version,
                    uri,
                    submitter: User,
                    uuid=None,
                    name=None,
                    hosting_service: models.WorkflowRegistry = None):
        if hosting_service:
            if self.external_id and hasattr(hosting_service,
                                            'get_external_uuid'):
                try:
                    self.uuid = hosting_service.get_external_uuid(
                        self.external_id, version, submitter)
                except RuntimeError as e:
                    raise lm_exceptions.NotAuthorizedException(details=str(e))
            elif not self.external_id and hasattr(hosting_service,
                                                  'get_external_id'):
                try:
                    self.external_id = hosting_service.get_external_id(
                        self.uuid, version, submitter)
                except lm_exceptions.EntityNotFoundException:
                    logger.warning(
                        "Unable to associate an external ID to the workflow")
        return WorkflowVersion(self,
                               uri,
                               version,
                               submitter,
                               uuid=uuid,
                               name=name,
                               hosting_service=hosting_service)

    def remove_version(self, version: WorkflowVersion):
        self.versions.remove(version)

    def get_user_versions(self,
                          user: models.User) -> List[models.WorkflowVersion]:
        return models.WorkflowVersion.query\
            .join(Permission, Permission.resource_id == models.WorkflowVersion.id)\
            .filter(models.WorkflowVersion.workflow_id == self.id)\
            .filter(Permission.user_id == user.id)\
            .all()

    def check_health(self) -> dict:
        health = {'healthy': True, 'issues': []}
        for suite in self.test_suites:
            for test_instance in suite.test_instances:
                try:
                    testing_service = test_instance.testing_service
                    if not testing_service.last_test_build.is_successful():
                        health["healthy"] = False
                except lm_exceptions.TestingServiceException as e:
                    health["issues"].append(str(e))
                    health["healthy"] = "Unknown"
        return health

    @classmethod
    def get_user_workflow(cls, owner: User, uuid) -> Workflow:
        try:
            return cls.query\
                .join(Permission)\
                .filter(Permission.resource_id == cls.id, Permission.user_id == owner.id)\
                .filter(cls.uuid == lm_utils.uuid_param(uuid)).one()
        except NoResultFound as e:
            logger.debug(e)
            return None
        except Exception as e:
            raise lm_exceptions.LifeMonitorException(detail=str(e),
                                                     stack=str(e))

    @classmethod
    def get_user_workflows(cls, owner: User) -> List[Workflow]:
        return cls.query.join(Permission)\
            .filter(Permission.user_id == owner.id).all()
Example #6
0
class TestingService(db.Model, ModelMixin):
    uuid = db.Column("uuid",
                     UUID,
                     db.ForeignKey(models.TestInstance.uuid),
                     primary_key=True)
    _type = db.Column("type", db.String, nullable=False)
    url = db.Column(db.Text, nullable=False, unique=True)
    _token = None

    # configure relationships
    test_instances = db.relationship("TestInstance",
                                     back_populates="testing_service")

    # configure the class manager
    service_type_registry = ClassManager('lifemonitor.api.models.services',
                                         class_suffix='TestingService',
                                         skip=['__init__', 'service'])

    __mapper_args__ = {
        'polymorphic_on': _type,
        'polymorphic_identity': 'testing_service'
    }

    def __init__(self,
                 url: str,
                 token: models.TestingServiceToken = None) -> None:
        self.url = url
        self._token = token

    def __repr__(self):
        return f'<TestingService {self.url}, ({self.uuid})>'

    @property
    def api_base_url(self):
        return self.url

    @property
    def token(self):
        if not self._token:
            logger.debug("Querying the token registry for the service %r...",
                         self.url)
            self._token = models.TestingServiceTokenManager.get_instance(
            ).get_token(self.url)
            if not self._token:
                logger.debug(
                    "Querying the token registry for the API service %r...",
                    self.api_base_url)
                self._token = models.TestingServiceTokenManager.get_instance(
                ).get_token(self.api_base_url)
        logger.debug("Set token for the testing service %r (type: %r): %r",
                     self.url, self._type, self._token is not None)
        return self._token

    def check_connection(self) -> bool:
        raise lm_exceptions.NotImplementedException()

    def is_workflow_healthy(self, test_instance: models.TestInstance) -> bool:
        raise lm_exceptions.NotImplementedException()

    def get_last_test_build(
            self, test_instance: models.TestInstance) -> models.TestBuild:
        raise lm_exceptions.NotImplementedException()

    def get_last_passed_test_build(
            self, test_instance: models.TestInstance) -> models.TestBuild:
        raise lm_exceptions.NotImplementedException()

    def get_last_failed_test_build(
            self, test_instance: models.TestInstance) -> models.TestBuild:
        raise lm_exceptions.NotImplementedException()

    def get_test_build(self, test_instance: models.TestInstance,
                       build_number) -> models.TestBuild:
        raise lm_exceptions.NotImplementedException()

    def get_test_builds(self,
                        test_instance: models.TestInstance,
                        limit=10) -> list:
        raise lm_exceptions.ßNotImplementedException()

    def get_test_builds_as_dict(self, test_instance: models.TestInstance,
                                test_output):
        last_test_build = self.last_test_build
        last_passed_test_build = self.last_passed_test_build
        last_failed_test_build = self.last_failed_test_build
        return {
            'last_test_build':
            last_test_build.to_dict(test_output) if last_test_build else None,
            'last_passed_test_build':
            last_passed_test_build.to_dict(test_output)
            if last_passed_test_build else None,
            'last_failed_test_build':
            last_failed_test_build.to_dict(test_output)
            if last_failed_test_build else None,
            "test_builds": [t.to_dict(test_output) for t in self.test_builds]
        }

    def to_dict(self, test_builds=False, test_output=False) -> dict:
        data = {
            'uuid': str(self.uuid),
            'testing_service_url': self.url,
            'workflow_healthy': self.is_workflow_healthy,
        }
        if test_builds:
            data["test_build"] = self.get_test_builds_as_dict(
                test_output=test_output)
        return data

    @classmethod
    def all(cls) -> List[TestingService]:
        return cls.query.all()

    @classmethod
    def find_by_uuid(cls, uuid) -> TestingService:
        return cls.query.get(uuid)

    @classmethod
    def find_by_url(cls, url) -> TestingService:
        try:
            return cls.query.filter(TestingService.url == url).one()
        except NoResultFound as e:
            logger.debug(e)
            return None
        except Exception as e:
            raise lm_exceptions.LifeMonitorException(detail=str(e),
                                                     stack=str(e))

    @classmethod
    def get_instance(cls, service_type, url: str) -> TestingService:
        try:
            # return the service obj if the service has already been registered
            instance = cls.find_by_url(url)
            logger.debug("Found service instance: %r", instance)
            if instance:
                return instance
            # try to instanciate the service if the it has not been registered yet
            return cls.service_type_registry.get_class(service_type)(url)
        except KeyError:
            raise lm_exceptions.TestingServiceNotSupportedException(
                f"Not supported testing service type '{service_type}'")
        except Exception as e:
            raise lm_exceptions.TestingServiceException(detail=str(e))
Example #7
0
class TestInstance(db.Model, ModelMixin):
    uuid = db.Column(UUID, primary_key=True, default=_uuid.uuid4)
    _test_suite_uuid = \
        db.Column("test_suite_uuid", UUID, db.ForeignKey(TestSuite.uuid), nullable=False)
    name = db.Column(db.Text, nullable=False)
    resource = db.Column(db.Text, nullable=False)
    parameters = db.Column(JSON, nullable=True)
    submitter_id = db.Column(db.Integer,
                             db.ForeignKey(models.User.id),
                             nullable=False)
    # configure relationships
    submitter = db.relationship("User", uselist=False)
    test_suite = db.relationship("TestSuite", back_populates="test_instances")
    testing_service = db.relationship(
        "TestingService",
        back_populates="test_instances",
        uselist=False,
        cascade="save-update, merge, delete, delete-orphan")

    def __init__(self, testing_suite: TestSuite, submitter: models.User,
                 test_name, test_resource,
                 testing_service: models.TestingService) -> None:
        self.test_suite = testing_suite
        self.submitter = submitter
        self.name = test_name
        self.resource = test_resource
        self.testing_service = testing_service

    def __repr__(self):
        return '<TestInstance {} on TestSuite {}>'.format(
            self.uuid, self.test_suite.uuid)

    @property
    def test(self):
        if not self.test_suite:
            raise EntityNotFoundException(models.Test)
        return self.test_suite.tests[self.name]

    @property
    def last_test_build(self):
        return self.testing_service.get_last_test_build(self)

    def get_test_builds(self, limit=10):
        return self.testing_service.get_test_builds(self, limit=limit)

    def get_test_build(self, build_number):
        return self.testing_service.get_test_build(self, build_number)

    def to_dict(self, test_build=False, test_output=False):
        data = {
            'uuid': str(self.uuid),
            'name': self.name,
            'parameters': self.parameters,
            'testing_service': self.testing_service.to_dict(test_builds=False)
        }
        if test_build:
            data.update(
                self.testing_service.get_test_builds_as_dict(
                    test_output=test_output))
        return data

    @classmethod
    def all(cls) -> List[TestInstance]:
        return cls.query.all()

    @classmethod
    def find_by_uuid(cls, uuid) -> TestInstance:
        return cls.query.get(uuid)
Example #8
0
class TestSuite(db.Model, ModelMixin):
    uuid = db.Column(UUID, primary_key=True, default=_uuid.uuid4)
    name = db.Column(db.String, nullable=True)
    roc_suite = db.Column(db.String, nullable=True)
    definition = db.Column(JSON, nullable=True)
    _workflow_version_id = db.Column("workflow_version_id", db.Integer,
                                     db.ForeignKey(models.workflows.WorkflowVersion.id), nullable=False)
    workflow_version = db.relationship("WorkflowVersion", back_populates="test_suites")
    submitter_id = db.Column(db.Integer,
                             db.ForeignKey(User.id), nullable=True)
    submitter = db.relationship("User", uselist=False)
    test_instances = db.relationship("TestInstance",
                                     back_populates="test_suite",
                                     cascade="all, delete-orphan")

    def __init__(self,
                 w: models.workflows.WorkflowVersion, submitter: User,
                 name: str = None, roc_suite: str = None, definition: object = None) -> None:
        self.workflow_version = w
        self.submitter = submitter
        self.name = name
        self.roc_suite = roc_suite
        self.definition = definition

    def __repr__(self):
        return '<TestSuite {} of workflow {} (version {})>'.format(
            self.uuid, self.workflow_version.uuid, self.workflow_version.version)

    @property
    def status(self) -> models.SuiteStatus:
        return models.SuiteStatus(self)

    def get_test_instance_by_name(self, name) -> list:
        result = []
        for ti in self.test_instances:
            if ti.name == name:
                result.append(ti)
        return result

    def to_dict(self, test_build=False, test_output=False) -> dict:
        return {
            'uuid': str(self.uuid),
            'test': [t.to_dict(test_build=test_build, test_output=test_output)
                     for t in self.test_instances]
        }

    def add_test_instance(self, submitter: User, managed: bool, test_name: str,
                          testing_service_type, testing_service_url, testing_service_resource,
                          roc_instance: str = None):
        testing_service = \
            models.TestingService.get_instance(testing_service_type, testing_service_url)
        assert testing_service, "Testing service not initialized"
        instance_cls = models.TestInstance if not managed else models.ManagedTestInstance
        test_instance = instance_cls(self, submitter, test_name, testing_service_resource, testing_service, roc_instance)
        logger.debug("Created TestInstance: %r", test_instance)
        return test_instance

    @classmethod
    def all(cls) -> List[TestSuite]:
        return cls.query.all()

    @classmethod
    def find_by_uuid(cls, uuid) -> TestSuite:
        return cls.query.get(uuid)
Example #9
0
class WorkflowRegistry(auth_models.Resource):

    id = db.Column(db.Integer,
                   db.ForeignKey(auth_models.Resource.id),
                   primary_key=True)
    registry_type = db.Column(db.String, nullable=False)
    _client_id = db.Column(
        db.Integer, db.ForeignKey('oauth2_client.id', ondelete='CASCADE'))
    _server_id = db.Column(
        db.Integer,
        db.ForeignKey('oauth2_identity_provider.id', ondelete='CASCADE'))
    client_credentials = db.relationship("Client",
                                         uselist=False,
                                         cascade="all, delete")
    server_credentials = db.relationship("OAuth2IdentityProvider",
                                         uselist=False,
                                         cascade="all, delete",
                                         foreign_keys=[_server_id],
                                         backref="workflow_registry")
    client_id = association_proxy('client_credentials', 'client_id')

    _client = None

    registry_types = ClassManager('lifemonitor.api.models.registries',
                                  class_suffix="WorkflowRegistry",
                                  skip=["registry"])

    __mapper_args__ = {'polymorphic_identity': 'workflow_registry'}

    def __init__(self, registry_type, client_credentials, server_credentials):
        super().__init__(server_credentials.api_base_url,
                         name=server_credentials.name)
        self.registry_type = registry_type
        self.client_credentials = client_credentials
        self.server_credentials = server_credentials
        self._client = None

    def __repr__(self):
        return '<WorkflowRegistry ({}) -- name {}, url {}>'.format(
            self.uuid, self.name, self.uri)

    @property
    def api(self) -> auth_models.Resource:
        return self.server_credentials.api_resource

    def set_name(self, name):
        self.name = name
        self.server_credentials.name = name

    def set_uri(self, uri):
        self.uri = uri
        self.server_credentials.api_resource.uri = uri
        self.client_credentials.api_base_url = uri

    def update_client(self,
                      client_id=None,
                      client_secret=None,
                      redirect_uris=None,
                      client_auth_method=None):
        if client_id:
            self.server_credentials.client_id = client_id
        if client_secret:
            self.server_credentials.client_secret = client_secret
        if redirect_uris:
            self.client_credentials.redirect_uris = redirect_uris
        if client_auth_method:
            self.client_credentials.auth_method = client_auth_method

    @property
    def client(self) -> WorkflowRegistryClient:
        if self._client is None:
            rtype = self.__class__.__name__.replace("WorkflowRegistry",
                                                    "").lower()
            return WorkflowRegistryClient.get_client_class(rtype)(self)
        return self._client

    def get_external_uuid(self, external_id, version,
                          user: auth_models.User) -> str:
        return self.client.get_external_uuid(external_id, version, user)

    def get_external_id(self, uuid, version, user: auth_models.User) -> str:
        return self.client.get_external_id(uuid, version, user)

    def build_ro_link(self, user, w: Union[models.WorkflowVersion,
                                           str]) -> str:
        return self.client.build_ro_link(user, w)

    def download_url(self, url, user, target_path=None):
        return self.client.download_url(url, user, target_path=target_path)

    @property
    def users(self) -> List[auth_models.User]:
        return self.get_users()

    @property
    def registered_workflow_versions(self) -> List[models.WorkflowVersion]:
        return self.ro_crates

    def get_authorization(self, user: auth_models.User):
        auths = auth_models.ExternalServiceAccessAuthorization.find_by_user_and_resource(
            user, self)
        # check for sub-resource authorizations
        return auths

    def get_user(self, user_id) -> auth_models.User:
        for u in self.users:
            logger.debug(f"Checking {u.id} {user_id}")
            if u.id == user_id:
                return u
        return None

    def get_users(self) -> List[auth_models.User]:
        try:
            return [
                i.user for i in OAuthIdentity.query.filter(
                    OAuthIdentity.provider == self.server_credentials).all()
            ]
        except Exception as e:
            raise lm_exceptions.EntityNotFoundException(e)

    def get_workflows(self) -> List[models.Workflow]:
        return list({w.workflow for w in self.registered_workflow_versions})

    def get_workflow(self, uuid_or_identifier) -> models.Workflow:
        try:
            w = next((
                w for w in self.registered_workflow_versions
                if w.workflow.uuid == lm_utils.uuid_param(uuid_or_identifier)),
                     None)
            return w.workflow if w is not None else None
        except ValueError:
            w = next((w for w in self.registered_workflow_versions
                      if w.workflow.external_id == uuid_or_identifier), None)
            return w.workflow if w is not None else None
        except Exception:
            if models.Workflow.find_by_uuid(uuid_or_identifier) is not None:
                raise lm_exceptions.NotAuthorizedException()

    def get_user_workflows(self,
                           user: auth_models.User) -> List[models.Workflow]:
        return self.client.filter_by_user(self.get_workflows(), user)

    def get_user_workflow_versions(
            self, user: auth_models.User) -> List[models.WorkflowVersion]:
        return self.client.filter_by_user(self.registered_workflow_versions,
                                          user)

    @classmethod
    def all(cls) -> List[WorkflowRegistry]:
        return cls.query.all()

    @classmethod
    def find_by_uuid(cls, uuid) -> WorkflowRegistry:
        try:
            return cls.query.filter(
                WorkflowRegistry.uuid == lm_utils.uuid_param(uuid)).one()
        except NoResultFound as e:
            logger.debug(e)
            return None
        except Exception as e:
            raise lm_exceptions.LifeMonitorException(detail=str(e),
                                                     stack=str(e))

    @classmethod
    def find_by_name(cls, name):
        try:
            return cls.query.filter(WorkflowRegistry.name == name).one()
        except Exception as e:
            raise lm_exceptions.EntityNotFoundException(WorkflowRegistry,
                                                        entity_id=name,
                                                        exception=e)

    @classmethod
    def find_by_uri(cls, uri):
        try:
            return cls.query.filter(WorkflowRegistry.uri == uri).one()
        except Exception as e:
            raise lm_exceptions.EntityNotFoundException(WorkflowRegistry,
                                                        entity_id=uri,
                                                        exception=e)

    @classmethod
    def find_by_client_id(cls, client_id):
        try:
            return cls.query.filter_by(client_id=client_id).one()
        except NoResultFound as e:
            logger.debug(e)
            return None
        except Exception as e:
            raise lm_exceptions.LifeMonitorException(detail=str(e),
                                                     stack=str(e))

    @classmethod
    def get_registry_class(cls, registry_type):
        return cls.registry_types.get_class(registry_type)

    @classmethod
    def new_instance(cls, registry_type, client_credentials,
                     server_credentials):
        return cls.get_registry_class(registry_type)(client_credentials,
                                                     server_credentials)
Example #10
0
class ROCrate(Resource):

    id = db.Column(db.Integer, db.ForeignKey(Resource.id), primary_key=True)
    hosting_service_id = db.Column(db.Integer,
                                   db.ForeignKey("resource.id"),
                                   nullable=True)
    hosting_service = db.relationship("Resource",
                                      uselist=False,
                                      backref=db.backref(
                                          "ro_crates",
                                          cascade="all, delete-orphan"),
                                      foreign_keys=[hosting_service_id])
    _metadata = db.Column("metadata", JSON, nullable=True)
    _test_metadata = None
    _local_path = None
    _metadata_loaded = False

    __mapper_args__ = {
        'polymorphic_identity': 'ro_crate',
        "inherit_condition": id == Resource.id
    }

    def __init__(self,
                 uri,
                 uuid=None,
                 name=None,
                 version=None,
                 hosting_service=None) -> None:
        super().__init__(uri, uuid=uuid, name=name, version=version)
        self.hosting_service = hosting_service
        self._crate_helper = None

    @hybrid_property
    def crate_metadata(self):
        if not self._metadata_loaded:
            self.load_metadata()
        return self._metadata

    @property
    def dataset_name(self):
        if not self._metadata_loaded:
            self.load_metadata()
        if not self._crate_helper:
            raise RuntimeError("ROCrate not correctly loaded")
        return self._crate_helper.name

    @property
    def test_metadata(self):
        if not self._metadata_loaded:
            self.load_metadata()
        return self._test_metadata

    def _get_authorizations(self):
        authorizations = self.authorizations.copy()
        authorizations.append(None)
        return authorizations

    def load_metadata(self):
        errors = []
        # try either with authorization hedaer and without authorization
        for authorization in self._get_authorizations():
            try:
                auth_header = authorization.as_http_header(
                ) if authorization else None
                logger.debug(auth_header)
                self._crate_helper, self._metadata, self._test_metadata = \
                    self.load_metadata_files(self.uri, authorization_header=auth_header)
                self._metadata_loaded = True
                return self._metadata, self._test_metadata
            except Exception as e:
                errors.append(e)

        if len([
                e for e in errors
                if isinstance(e, lm_exceptions.NotAuthorizedException)
        ]) == len(errors):
            raise lm_exceptions.NotAuthorizedException()
        raise lm_exceptions.LifeMonitorException("ROCrate download error",
                                                 errors=errors)

    @staticmethod
    def extract_rocrate(roc_link, target_path=None, authorization_header=None):
        with tempfile.NamedTemporaryFile(dir="/tmp") as archive_path:
            zip_archive = download_url(roc_link,
                                       target_path=archive_path.name,
                                       authorization=authorization_header)
            logger.debug("ZIP Archive: %s", zip_archive)
            roc_path = target_path or Path(tempfile.mkdtemp(dir="/tmp"))
            logger.info("Extracting RO Crate @ %s", roc_path)
            extract_zip(archive_path, target_path=roc_path.as_posix())
            return roc_path

    @classmethod
    def load_metadata_files(cls, roc_link, authorization_header=None):
        roc_path = cls.extract_rocrate(
            roc_link, authorization_header=authorization_header)
        try:
            roc_posix_path = roc_path.as_posix()
            logger.debug(os.listdir(roc_posix_path))
            crate = ROCrateHelper(roc_posix_path)
            metadata_path = Path(roc_posix_path) / crate.metadata.id
            with open(metadata_path, "rt") as f:
                metadata = json.load(f)
            # create a new Workflow instance with the loaded metadata
            test_metadata = get_old_format_tests(crate)
            return crate, metadata, test_metadata
        finally:
            shutil.rmtree(roc_path, ignore_errors=True)