class ROCrate(Resource):
id = db.Column(db.Integer, db.ForeignKey(Resource.id), primary_key=True)
hosting_service_id = db.Column(db.Integer, db.ForeignKey("resource.id"), nullable=True)
hosting_service = db.relationship("Resource", uselist=False,
backref=db.backref("ro_crates", cascade="all, delete-orphan"),
foreign_keys=[hosting_service_id])
_metadata = db.Column("metadata", JSON, nullable=True)
_local_path = None
_metadata_loaded = False
__roc_helper = None
__mapper_args__ = {
'polymorphic_identity': 'ro_crate',
"inherit_condition": id == Resource.id
}
def __init__(self, uri, uuid=None, name=None,
version=None, hosting_service=None) -> None:
super().__init__(uri, uuid=uuid, name=name, version=version)
self.hosting_service = hosting_service
self.__roc_helper = None
@hybrid_property
def crate_metadata(self):
if not self._metadata_loaded:
self.load_metadata()
return self._metadata
@hybrid_property
def roc_suites(self):
return get_roc_suites(self._roc_helper)
def get_roc_suite(self, roc_suite_identifier):
try:
return self.roc_suites[roc_suite_identifier]
except KeyError:
logger.warning("Unable to find the roc_suite with identifier: %r", roc_suite_identifier)
return None
@property
def dataset_name(self):
return self._roc_helper.name
@property
def _roc_helper(self):
if not self.__roc_helper:
if not self._metadata_loaded:
self.load_metadata()
if not self.__roc_helper:
raise RuntimeError("ROCrate not correctly loaded")
return self.__roc_helper
def _get_authorizations(self):
authorizations = self.authorizations.copy()
authorizations.append(None)
return authorizations
def load_metadata(self) -> dict:
errors = []
# try either with authorization header and without authorization
for authorization in self._get_authorizations():
try:
auth_header = authorization.as_http_header() if authorization else None
logger.debug(auth_header)
self.__roc_helper, self._metadata = \
self.load_metadata_files(self.uri, authorization_header=auth_header)
self._metadata_loaded = True
return self._metadata
except lm_exceptions.NotAuthorizedException as e:
logger.info("Caught authorization error exception while downloading and processing RO-crate: %s", e)
errors.append(str(e))
raise lm_exceptions.NotAuthorizedException(detail=f"Not authorized to download {self.uri}", original_errors=errors)
@classmethod
def load_metadata_files(cls, roc_link, authorization_header=None):
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir_path = Path(tmpdir)
local_zip = download_url(roc_link,
target_path=(tmpdir_path / 'rocrate.zip').as_posix(),
authorization=authorization_header)
logger.debug("ZIP Archive: %s", local_zip)
extracted_roc_path = tmpdir_path / 'extracted-rocrate'
logger.info("Extracting RO Crate to %s", extracted_roc_path)
extract_zip(local_zip, target_path=extracted_roc_path.as_posix())
if logger.isEnabledFor(logging.DEBUG):
logger.debug("RO-crate contents: %s", ', '.join(str(s) for s in extracted_roc_path.iterdir()))
try:
crate = ROCrateHelper(extracted_roc_path.as_posix())
metadata_path = extracted_roc_path / crate.metadata.id
with open(metadata_path, "rt") as f:
metadata = json.load(f)
return crate, metadata
except Exception as e:
raise lm_exceptions.NotValidROCrateException(detail=str(e))
class TestSuite(db.Model, ModelMixin):
uuid = db.Column(UUID, primary_key=True, default=_uuid.uuid4)
_workflow_version_id = db.Column("workflow_version_id",
db.Integer,
db.ForeignKey(
models.workflows.WorkflowVersion.id),
nullable=False)
workflow_version = db.relationship("WorkflowVersion",
back_populates="test_suites")
test_definition = db.Column(JSON, nullable=False)
submitter_id = db.Column(db.Integer,
db.ForeignKey(User.id),
nullable=False)
submitter = db.relationship("User", uselist=False)
test_instances = db.relationship("TestInstance",
back_populates="test_suite",
cascade="all, delete")
def __init__(self, w: models.workflows.WorkflowVersion, submitter: User,
test_definition: object) -> None:
self.workflow_version = w
self.submitter = submitter
self.test_definition = test_definition
self._parse_test_definition()
def __repr__(self):
return '<TestSuite {} of workflow {} (version {})>'.format(
self.uuid, self.workflow_version.uuid,
self.workflow_version.version)
def _parse_test_definition(self):
try:
for test in self.test_definition["test"]:
test = tm.Test.from_json(test)
for instance in test.instance:
logger.debug("Instance: %r", instance)
testing_service = models.TestingService.get_instance(
instance.service.type, instance.service.url)
assert testing_service, "Testing service not initialized"
logger.debug("Created TestService: %r", testing_service)
test_instance = models.TestInstance(
self, self.submitter, test.name,
instance.service.resource, testing_service)
logger.debug("Created TestInstance: %r", test_instance)
except KeyError as e:
raise lm_exceptions.SpecificationNotValidException(
f"Missing property: {e}")
@property
def status(self) -> models.SuiteStatus:
return models.SuiteStatus(self)
def get_test_instance_by_name(self, name) -> list:
result = []
for ti in self.test_instances:
if ti.name == name:
result.append(ti)
return result
def to_dict(self, test_build=False, test_output=False) -> dict:
return {
'uuid':
str(self.uuid),
'test': [
t.to_dict(test_build=test_build, test_output=test_output)
for t in self.test_instances
]
}
def add_test_instance(self, submitter: User, test_name,
testing_service_type, testing_service_url,
testing_service_resource):
testing_service = \
models.TestingService.get_instance(testing_service_type, testing_service_url)
test_instance = models.TestInstance(self, submitter, test_name,
testing_service_resource,
testing_service)
logger.debug("Created TestInstance: %r", test_instance)
return test_instance
@property
def tests(self) -> Optional[dict]:
if not self.test_definition:
raise lm_exceptions.SpecificationNotDefinedException(
'Not test definition for the test suite {}'.format(self.uuid))
if "test" not in self.test_definition:
raise lm_exceptions.SpecificationNotValidException(
"'test' property not found")
# TODO: implement a caching mechanism: with a custom setter for the test_definition collection
result = {}
for test in self.test_definition["test"]:
result[test["name"]] = Test(
self, test["name"],
test["specification"] if "specification" in test else None)
return result
@classmethod
def all(cls) -> List[TestSuite]:
return cls.query.all()
@classmethod
def find_by_uuid(cls, uuid) -> TestSuite:
return cls.query.get(uuid)
class TestInstance(db.Model, ModelMixin):
uuid = db.Column(UUID, primary_key=True, default=_uuid.uuid4)
type = db.Column(db.String(20), nullable=False)
_test_suite_uuid = \
db.Column("test_suite_uuid", UUID, db.ForeignKey(TestSuite.uuid), nullable=False)
name = db.Column(db.String, nullable=False)
roc_instance = db.Column(db.String, nullable=True)
resource = db.Column(db.Text, nullable=False)
parameters = db.Column(JSON, nullable=True)
submitter_id = db.Column(db.Integer,
db.ForeignKey(models.User.id), nullable=True)
# configure relationships
submitter = db.relationship("User", uselist=False)
test_suite = db.relationship("TestSuite",
back_populates="test_instances",
foreign_keys=[_test_suite_uuid])
testing_service_id = db.Column(UUID, db.ForeignKey("testing_service.uuid"), nullable=False)
testing_service = db.relationship("TestingService",
foreign_keys=[testing_service_id],
backref=db.backref("test_instances", cascade="all, delete-orphan"),
uselist=False)
__mapper_args__ = {
'polymorphic_on': type,
'polymorphic_identity': 'unmanaged'
}
def __init__(self, testing_suite: TestSuite, submitter: models.User,
test_name, test_resource, testing_service: models.TestingService,
roc_instance: str = None) -> None:
self.test_suite = testing_suite
self.submitter = submitter
self.name = test_name
self.roc_instance = roc_instance
self.resource = test_resource
self.testing_service = testing_service
def __repr__(self):
return '<TestInstance {} on TestSuite {}>'.format(self.uuid, self.test_suite.uuid)
@property
def is_roc_instance(self):
return self.roc_instance is not None
@property
def managed(self):
return self.type != 'unmanaged'
@property
def last_test_build(self):
return self.testing_service.get_last_test_build(self)
def get_test_builds(self, limit=10):
return self.testing_service.get_test_builds(self, limit=limit)
def get_test_build(self, build_number):
return self.testing_service.get_test_build(self, build_number)
def to_dict(self, test_build=False, test_output=False):
data = {
'uuid': str(self.uuid),
'name': self.name,
'parameters': self.parameters,
'testing_service': self.testing_service.to_dict(test_builds=False)
}
if test_build:
data.update(self.testing_service.get_test_builds_as_dict(test_output=test_output))
return data
@classmethod
def all(cls) -> List[TestInstance]:
return cls.query.all()
@classmethod
def find_by_uuid(cls, uuid) -> TestInstance:
return cls.query.get(uuid)
class WorkflowVersion(ROCrate):
id = db.Column(db.Integer, db.ForeignKey(ROCrate.id), primary_key=True)
submitter_id = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
workflow_id = \
db.Column(db.Integer, db.ForeignKey("workflow.id"), nullable=False)
workflow = db.relationship(
"Workflow",
foreign_keys=[workflow_id],
cascade="all",
backref=db.backref(
"versions",
cascade="all, delete-orphan",
collection_class=attribute_mapped_collection('version')))
test_suites = db.relationship("TestSuite",
back_populates="workflow_version",
cascade="all, delete")
submitter = db.relationship(
"User",
uselist=False,
backref=db.backref("workflows",
cascade="all, delete-orphan",
collection_class=WorkflowVersionCollection))
roc_link = association_proxy('ro_crate', 'uri')
__mapper_args__ = {'polymorphic_identity': 'workflow_version'}
def __init__(self,
workflow: Workflow,
uri,
version,
submitter: User,
uuid=None,
name=None,
hosting_service: models.WorkflowRegistry = None) -> None:
super().__init__(uri,
uuid=uuid,
name=name,
version=version,
hosting_service=hosting_service)
self.submitter = submitter
self.workflow = workflow
def __repr__(self):
return '<WorkflowVersion ({}, {}), name: {}, ro_crate link {}>'.format(
self.uuid, self.version, self.name, self.roc_link)
def check_health(self) -> dict:
health = {'healthy': True, 'issues': []}
for suite in self.test_suites:
for test_instance in suite.test_instances:
try:
testing_service = test_instance.testing_service
if not testing_service.last_test_build.is_successful():
health["healthy"] = False
except lm_exceptions.TestingServiceException as e:
health["issues"].append(str(e))
health["healthy"] = "Unknown"
return health
@hybrid_property
def authorizations(self):
auths = [a for a in self._authorizations]
if self.hosting_service and self.submitter:
for auth in self.submitter.get_authorization(self.hosting_service):
auths.append(auth)
return auths
@hybrid_property
def workflow_registry(self) -> models.WorkflowRegistry:
return self.hosting_service
@hybrid_property
def roc_link(self) -> str:
return self.uri
@property
def is_latest(self) -> bool:
return self.workflow.latest_version.version == self.version
@property
def previous_versions(self) -> List[str]:
return [
w.version for w in self.workflow.versions.values()
if w != self and w.version < self.version
]
@property
def previous_workflow_versions(self) -> List[models.WorkflowVersion]:
return [
w for w in self.workflow.versions.values()
if w != self and w.version < self.version
]
@property
def status(self) -> models.WorkflowStatus:
return models.WorkflowStatus(self)
@property
def is_healthy(self) -> Union[bool, str]:
return self.check_health()["healthy"]
def add_test_suite(self,
submitter: User,
name: str = None,
roc_suite: str = None,
definition: object = None):
return models.TestSuite(self,
submitter,
name=name,
roc_suite=roc_suite,
definition=definition)
@property
def submitter_identity(self):
# Return the submitter identity wrt the registry
identity = OAuthIdentity.find_by_user_id(self.submitter.id,
self.workflow_registry.name)
return identity.provider_user_id
def to_dict(self, test_suite=False, test_build=False, test_output=False):
health = self.check_health()
data = {
'uuid': str(self.uuid),
'version': self.version,
'name': self.name,
'roc_link': self.roc_link,
'isHealthy': health["healthy"],
'issues': health["issues"]
}
if test_suite:
data['test_suite'] = [
s.to_dict(test_build=test_build, test_output=test_output)
for s in self.test_suites
]
return data
def delete(self):
if len(self.workflow.versions) > 1:
workflow = self.workflow
self.workflow.remove_version(self)
workflow.save()
else:
self.workflow.delete()
@classmethod
def all(cls) -> List[WorkflowVersion]:
return cls.query.all()
@classmethod
def get_submitter_versions(cls, submitter: User) -> List[WorkflowVersion]:
return cls.query.filter(
WorkflowVersion.submitter_id == submitter.id).all()
@classmethod
def get_user_workflow_version(cls, owner: User, uuid,
version) -> WorkflowVersion:
try:
return cls.query\
.join(Workflow, Workflow.id == cls.workflow_id)\
.join(Permission, Permission.resource_id == cls.id)\
.filter(Workflow.uuid == lm_utils.uuid_param(uuid))\
.filter(Permission.user_id == owner.id)\
.filter(cls.version == version).one()
except NoResultFound as e:
logger.exception(e)
return None
except Exception as e:
raise lm_exceptions.LifeMonitorException(detail=str(e),
stack=str(e))
@classmethod
def get_user_workflow_versions(cls, owner: User) -> List[WorkflowVersion]:
return cls.query\
.join(Permission)\
.filter(Permission.resource_id == cls.id, Permission.user_id == owner.id).all()
@classmethod
def get_hosted_workflow_version(cls, hosting_service: Resource, uuid,
version) -> List[WorkflowVersion]:
# TODO: replace WorkflowRegistry with a more general Entity
try:
return cls.query\
.join(WorkflowRegistry, cls.hosting_service)\
.join(Workflow, Workflow.id == cls.workflow_id)\
.filter(WorkflowRegistry.uuid == lm_utils.uuid_param(hosting_service.uuid))\
.filter(Workflow.uuid == lm_utils.uuid_param(uuid))\
.filter(cls.version == version)\
.order_by(WorkflowVersion.version.desc()).one()
except NoResultFound as e:
logger.debug(e)
return None
except Exception as e:
raise lm_exceptions.LifeMonitorException(detail=str(e),
stack=str(e))
@classmethod
def get_hosted_workflow_versions(
cls, hosting_service: Resource) -> List[WorkflowVersion]:
# TODO: replace WorkflowRegistry with a more general Entity
return cls.query\
.join(WorkflowRegistry, cls.hosting_service)\
.filter(WorkflowRegistry.uuid == lm_utils.uuid_param(hosting_service.uuid))\
.order_by(WorkflowVersion.version.desc()).all()
class Workflow(Resource):
id = db.Column(db.Integer, db.ForeignKey(Resource.id), primary_key=True)
external_ns = "external-id:"
__mapper_args__ = {'polymorphic_identity': 'workflow'}
def __init__(self,
uri=None,
uuid=None,
identifier=None,
version=None,
name=None) -> None:
super().__init__(uri=uri or f"{self.external_ns}",
uuid=uuid,
version=version,
name=name)
if identifier is not None:
self.external_id = identifier
def __repr__(self):
return '<Workflow ({}), name: {}>'.format(self.uuid, self.name)
@hybrid_property
def external_id(self):
r = self.uri.replace(self.external_ns, "")
return r if len(r) > 0 else None
@external_id.setter
def external_id(self, value):
self.uri = f"{self.external_ns}{value}"
@hybrid_property
def latest_version(self) -> WorkflowVersion:
return max(self.versions.values(), key=lambda v: v.version)
def add_version(self,
version,
uri,
submitter: User,
uuid=None,
name=None,
hosting_service: models.WorkflowRegistry = None):
if hosting_service:
if self.external_id and hasattr(hosting_service,
'get_external_uuid'):
try:
self.uuid = hosting_service.get_external_uuid(
self.external_id, version, submitter)
except RuntimeError as e:
raise lm_exceptions.NotAuthorizedException(details=str(e))
elif not self.external_id and hasattr(hosting_service,
'get_external_id'):
try:
self.external_id = hosting_service.get_external_id(
self.uuid, version, submitter)
except lm_exceptions.EntityNotFoundException:
logger.warning(
"Unable to associate an external ID to the workflow")
return WorkflowVersion(self,
uri,
version,
submitter,
uuid=uuid,
name=name,
hosting_service=hosting_service)
def remove_version(self, version: WorkflowVersion):
self.versions.remove(version)
def get_user_versions(self,
user: models.User) -> List[models.WorkflowVersion]:
return models.WorkflowVersion.query\
.join(Permission, Permission.resource_id == models.WorkflowVersion.id)\
.filter(models.WorkflowVersion.workflow_id == self.id)\
.filter(Permission.user_id == user.id)\
.all()
def check_health(self) -> dict:
health = {'healthy': True, 'issues': []}
for suite in self.test_suites:
for test_instance in suite.test_instances:
try:
testing_service = test_instance.testing_service
if not testing_service.last_test_build.is_successful():
health["healthy"] = False
except lm_exceptions.TestingServiceException as e:
health["issues"].append(str(e))
health["healthy"] = "Unknown"
return health
@classmethod
def get_user_workflow(cls, owner: User, uuid) -> Workflow:
try:
return cls.query\
.join(Permission)\
.filter(Permission.resource_id == cls.id, Permission.user_id == owner.id)\
.filter(cls.uuid == lm_utils.uuid_param(uuid)).one()
except NoResultFound as e:
logger.debug(e)
return None
except Exception as e:
raise lm_exceptions.LifeMonitorException(detail=str(e),
stack=str(e))
@classmethod
def get_user_workflows(cls, owner: User) -> List[Workflow]:
return cls.query.join(Permission)\
.filter(Permission.user_id == owner.id).all()
class TestingService(db.Model, ModelMixin):
uuid = db.Column("uuid",
UUID,
db.ForeignKey(models.TestInstance.uuid),
primary_key=True)
_type = db.Column("type", db.String, nullable=False)
url = db.Column(db.Text, nullable=False, unique=True)
_token = None
# configure relationships
test_instances = db.relationship("TestInstance",
back_populates="testing_service")
# configure the class manager
service_type_registry = ClassManager('lifemonitor.api.models.services',
class_suffix='TestingService',
skip=['__init__', 'service'])
__mapper_args__ = {
'polymorphic_on': _type,
'polymorphic_identity': 'testing_service'
}
def __init__(self,
url: str,
token: models.TestingServiceToken = None) -> None:
self.url = url
self._token = token
def __repr__(self):
return f'<TestingService {self.url}, ({self.uuid})>'
@property
def api_base_url(self):
return self.url
@property
def token(self):
if not self._token:
logger.debug("Querying the token registry for the service %r...",
self.url)
self._token = models.TestingServiceTokenManager.get_instance(
).get_token(self.url)
if not self._token:
logger.debug(
"Querying the token registry for the API service %r...",
self.api_base_url)
self._token = models.TestingServiceTokenManager.get_instance(
).get_token(self.api_base_url)
logger.debug("Set token for the testing service %r (type: %r): %r",
self.url, self._type, self._token is not None)
return self._token
def check_connection(self) -> bool:
raise lm_exceptions.NotImplementedException()
def is_workflow_healthy(self, test_instance: models.TestInstance) -> bool:
raise lm_exceptions.NotImplementedException()
def get_last_test_build(
self, test_instance: models.TestInstance) -> models.TestBuild:
raise lm_exceptions.NotImplementedException()
def get_last_passed_test_build(
self, test_instance: models.TestInstance) -> models.TestBuild:
raise lm_exceptions.NotImplementedException()
def get_last_failed_test_build(
self, test_instance: models.TestInstance) -> models.TestBuild:
raise lm_exceptions.NotImplementedException()
def get_test_build(self, test_instance: models.TestInstance,
build_number) -> models.TestBuild:
raise lm_exceptions.NotImplementedException()
def get_test_builds(self,
test_instance: models.TestInstance,
limit=10) -> list:
raise lm_exceptions.ßNotImplementedException()
def get_test_builds_as_dict(self, test_instance: models.TestInstance,
test_output):
last_test_build = self.last_test_build
last_passed_test_build = self.last_passed_test_build
last_failed_test_build = self.last_failed_test_build
return {
'last_test_build':
last_test_build.to_dict(test_output) if last_test_build else None,
'last_passed_test_build':
last_passed_test_build.to_dict(test_output)
if last_passed_test_build else None,
'last_failed_test_build':
last_failed_test_build.to_dict(test_output)
if last_failed_test_build else None,
"test_builds": [t.to_dict(test_output) for t in self.test_builds]
}
def to_dict(self, test_builds=False, test_output=False) -> dict:
data = {
'uuid': str(self.uuid),
'testing_service_url': self.url,
'workflow_healthy': self.is_workflow_healthy,
}
if test_builds:
data["test_build"] = self.get_test_builds_as_dict(
test_output=test_output)
return data
@classmethod
def all(cls) -> List[TestingService]:
return cls.query.all()
@classmethod
def find_by_uuid(cls, uuid) -> TestingService:
return cls.query.get(uuid)
@classmethod
def find_by_url(cls, url) -> TestingService:
try:
return cls.query.filter(TestingService.url == url).one()
except NoResultFound as e:
logger.debug(e)
return None
except Exception as e:
raise lm_exceptions.LifeMonitorException(detail=str(e),
stack=str(e))
@classmethod
def get_instance(cls, service_type, url: str) -> TestingService:
try:
# return the service obj if the service has already been registered
instance = cls.find_by_url(url)
logger.debug("Found service instance: %r", instance)
if instance:
return instance
# try to instanciate the service if the it has not been registered yet
return cls.service_type_registry.get_class(service_type)(url)
except KeyError:
raise lm_exceptions.TestingServiceNotSupportedException(
f"Not supported testing service type '{service_type}'")
except Exception as e:
raise lm_exceptions.TestingServiceException(detail=str(e))
class TestInstance(db.Model, ModelMixin):
uuid = db.Column(UUID, primary_key=True, default=_uuid.uuid4)
_test_suite_uuid = \
db.Column("test_suite_uuid", UUID, db.ForeignKey(TestSuite.uuid), nullable=False)
name = db.Column(db.Text, nullable=False)
resource = db.Column(db.Text, nullable=False)
parameters = db.Column(JSON, nullable=True)
submitter_id = db.Column(db.Integer,
db.ForeignKey(models.User.id),
nullable=False)
# configure relationships
submitter = db.relationship("User", uselist=False)
test_suite = db.relationship("TestSuite", back_populates="test_instances")
testing_service = db.relationship(
"TestingService",
back_populates="test_instances",
uselist=False,
cascade="save-update, merge, delete, delete-orphan")
def __init__(self, testing_suite: TestSuite, submitter: models.User,
test_name, test_resource,
testing_service: models.TestingService) -> None:
self.test_suite = testing_suite
self.submitter = submitter
self.name = test_name
self.resource = test_resource
self.testing_service = testing_service
def __repr__(self):
return '<TestInstance {} on TestSuite {}>'.format(
self.uuid, self.test_suite.uuid)
@property
def test(self):
if not self.test_suite:
raise EntityNotFoundException(models.Test)
return self.test_suite.tests[self.name]
@property
def last_test_build(self):
return self.testing_service.get_last_test_build(self)
def get_test_builds(self, limit=10):
return self.testing_service.get_test_builds(self, limit=limit)
def get_test_build(self, build_number):
return self.testing_service.get_test_build(self, build_number)
def to_dict(self, test_build=False, test_output=False):
data = {
'uuid': str(self.uuid),
'name': self.name,
'parameters': self.parameters,
'testing_service': self.testing_service.to_dict(test_builds=False)
}
if test_build:
data.update(
self.testing_service.get_test_builds_as_dict(
test_output=test_output))
return data
@classmethod
def all(cls) -> List[TestInstance]:
return cls.query.all()
@classmethod
def find_by_uuid(cls, uuid) -> TestInstance:
return cls.query.get(uuid)
class TestSuite(db.Model, ModelMixin):
uuid = db.Column(UUID, primary_key=True, default=_uuid.uuid4)
name = db.Column(db.String, nullable=True)
roc_suite = db.Column(db.String, nullable=True)
definition = db.Column(JSON, nullable=True)
_workflow_version_id = db.Column("workflow_version_id", db.Integer,
db.ForeignKey(models.workflows.WorkflowVersion.id), nullable=False)
workflow_version = db.relationship("WorkflowVersion", back_populates="test_suites")
submitter_id = db.Column(db.Integer,
db.ForeignKey(User.id), nullable=True)
submitter = db.relationship("User", uselist=False)
test_instances = db.relationship("TestInstance",
back_populates="test_suite",
cascade="all, delete-orphan")
def __init__(self,
w: models.workflows.WorkflowVersion, submitter: User,
name: str = None, roc_suite: str = None, definition: object = None) -> None:
self.workflow_version = w
self.submitter = submitter
self.name = name
self.roc_suite = roc_suite
self.definition = definition
def __repr__(self):
return '<TestSuite {} of workflow {} (version {})>'.format(
self.uuid, self.workflow_version.uuid, self.workflow_version.version)
@property
def status(self) -> models.SuiteStatus:
return models.SuiteStatus(self)
def get_test_instance_by_name(self, name) -> list:
result = []
for ti in self.test_instances:
if ti.name == name:
result.append(ti)
return result
def to_dict(self, test_build=False, test_output=False) -> dict:
return {
'uuid': str(self.uuid),
'test': [t.to_dict(test_build=test_build, test_output=test_output)
for t in self.test_instances]
}
def add_test_instance(self, submitter: User, managed: bool, test_name: str,
testing_service_type, testing_service_url, testing_service_resource,
roc_instance: str = None):
testing_service = \
models.TestingService.get_instance(testing_service_type, testing_service_url)
assert testing_service, "Testing service not initialized"
instance_cls = models.TestInstance if not managed else models.ManagedTestInstance
test_instance = instance_cls(self, submitter, test_name, testing_service_resource, testing_service, roc_instance)
logger.debug("Created TestInstance: %r", test_instance)
return test_instance
@classmethod
def all(cls) -> List[TestSuite]:
return cls.query.all()
@classmethod
def find_by_uuid(cls, uuid) -> TestSuite:
return cls.query.get(uuid)
class WorkflowRegistry(auth_models.Resource):
id = db.Column(db.Integer,
db.ForeignKey(auth_models.Resource.id),
primary_key=True)
registry_type = db.Column(db.String, nullable=False)
_client_id = db.Column(
db.Integer, db.ForeignKey('oauth2_client.id', ondelete='CASCADE'))
_server_id = db.Column(
db.Integer,
db.ForeignKey('oauth2_identity_provider.id', ondelete='CASCADE'))
client_credentials = db.relationship("Client",
uselist=False,
cascade="all, delete")
server_credentials = db.relationship("OAuth2IdentityProvider",
uselist=False,
cascade="all, delete",
foreign_keys=[_server_id],
backref="workflow_registry")
client_id = association_proxy('client_credentials', 'client_id')
_client = None
registry_types = ClassManager('lifemonitor.api.models.registries',
class_suffix="WorkflowRegistry",
skip=["registry"])
__mapper_args__ = {'polymorphic_identity': 'workflow_registry'}
def __init__(self, registry_type, client_credentials, server_credentials):
super().__init__(server_credentials.api_base_url,
name=server_credentials.name)
self.registry_type = registry_type
self.client_credentials = client_credentials
self.server_credentials = server_credentials
self._client = None
def __repr__(self):
return '<WorkflowRegistry ({}) -- name {}, url {}>'.format(
self.uuid, self.name, self.uri)
@property
def api(self) -> auth_models.Resource:
return self.server_credentials.api_resource
def set_name(self, name):
self.name = name
self.server_credentials.name = name
def set_uri(self, uri):
self.uri = uri
self.server_credentials.api_resource.uri = uri
self.client_credentials.api_base_url = uri
def update_client(self,
client_id=None,
client_secret=None,
redirect_uris=None,
client_auth_method=None):
if client_id:
self.server_credentials.client_id = client_id
if client_secret:
self.server_credentials.client_secret = client_secret
if redirect_uris:
self.client_credentials.redirect_uris = redirect_uris
if client_auth_method:
self.client_credentials.auth_method = client_auth_method
@property
def client(self) -> WorkflowRegistryClient:
if self._client is None:
rtype = self.__class__.__name__.replace("WorkflowRegistry",
"").lower()
return WorkflowRegistryClient.get_client_class(rtype)(self)
return self._client
def get_external_uuid(self, external_id, version,
user: auth_models.User) -> str:
return self.client.get_external_uuid(external_id, version, user)
def get_external_id(self, uuid, version, user: auth_models.User) -> str:
return self.client.get_external_id(uuid, version, user)
def build_ro_link(self, user, w: Union[models.WorkflowVersion,
str]) -> str:
return self.client.build_ro_link(user, w)
def download_url(self, url, user, target_path=None):
return self.client.download_url(url, user, target_path=target_path)
@property
def users(self) -> List[auth_models.User]:
return self.get_users()
@property
def registered_workflow_versions(self) -> List[models.WorkflowVersion]:
return self.ro_crates
def get_authorization(self, user: auth_models.User):
auths = auth_models.ExternalServiceAccessAuthorization.find_by_user_and_resource(
user, self)
# check for sub-resource authorizations
return auths
def get_user(self, user_id) -> auth_models.User:
for u in self.users:
logger.debug(f"Checking {u.id} {user_id}")
if u.id == user_id:
return u
return None
def get_users(self) -> List[auth_models.User]:
try:
return [
i.user for i in OAuthIdentity.query.filter(
OAuthIdentity.provider == self.server_credentials).all()
]
except Exception as e:
raise lm_exceptions.EntityNotFoundException(e)
def get_workflows(self) -> List[models.Workflow]:
return list({w.workflow for w in self.registered_workflow_versions})
def get_workflow(self, uuid_or_identifier) -> models.Workflow:
try:
w = next((
w for w in self.registered_workflow_versions
if w.workflow.uuid == lm_utils.uuid_param(uuid_or_identifier)),
None)
return w.workflow if w is not None else None
except ValueError:
w = next((w for w in self.registered_workflow_versions
if w.workflow.external_id == uuid_or_identifier), None)
return w.workflow if w is not None else None
except Exception:
if models.Workflow.find_by_uuid(uuid_or_identifier) is not None:
raise lm_exceptions.NotAuthorizedException()
def get_user_workflows(self,
user: auth_models.User) -> List[models.Workflow]:
return self.client.filter_by_user(self.get_workflows(), user)
def get_user_workflow_versions(
self, user: auth_models.User) -> List[models.WorkflowVersion]:
return self.client.filter_by_user(self.registered_workflow_versions,
user)
@classmethod
def all(cls) -> List[WorkflowRegistry]:
return cls.query.all()
@classmethod
def find_by_uuid(cls, uuid) -> WorkflowRegistry:
try:
return cls.query.filter(
WorkflowRegistry.uuid == lm_utils.uuid_param(uuid)).one()
except NoResultFound as e:
logger.debug(e)
return None
except Exception as e:
raise lm_exceptions.LifeMonitorException(detail=str(e),
stack=str(e))
@classmethod
def find_by_name(cls, name):
try:
return cls.query.filter(WorkflowRegistry.name == name).one()
except Exception as e:
raise lm_exceptions.EntityNotFoundException(WorkflowRegistry,
entity_id=name,
exception=e)
@classmethod
def find_by_uri(cls, uri):
try:
return cls.query.filter(WorkflowRegistry.uri == uri).one()
except Exception as e:
raise lm_exceptions.EntityNotFoundException(WorkflowRegistry,
entity_id=uri,
exception=e)
@classmethod
def find_by_client_id(cls, client_id):
try:
return cls.query.filter_by(client_id=client_id).one()
except NoResultFound as e:
logger.debug(e)
return None
except Exception as e:
raise lm_exceptions.LifeMonitorException(detail=str(e),
stack=str(e))
@classmethod
def get_registry_class(cls, registry_type):
return cls.registry_types.get_class(registry_type)
@classmethod
def new_instance(cls, registry_type, client_credentials,
server_credentials):
return cls.get_registry_class(registry_type)(client_credentials,
server_credentials)
class ROCrate(Resource):
id = db.Column(db.Integer, db.ForeignKey(Resource.id), primary_key=True)
hosting_service_id = db.Column(db.Integer,
db.ForeignKey("resource.id"),
nullable=True)
hosting_service = db.relationship("Resource",
uselist=False,
backref=db.backref(
"ro_crates",
cascade="all, delete-orphan"),
foreign_keys=[hosting_service_id])
_metadata = db.Column("metadata", JSON, nullable=True)
_test_metadata = None
_local_path = None
_metadata_loaded = False
__mapper_args__ = {
'polymorphic_identity': 'ro_crate',
"inherit_condition": id == Resource.id
}
def __init__(self,
uri,
uuid=None,
name=None,
version=None,
hosting_service=None) -> None:
super().__init__(uri, uuid=uuid, name=name, version=version)
self.hosting_service = hosting_service
self._crate_helper = None
@hybrid_property
def crate_metadata(self):
if not self._metadata_loaded:
self.load_metadata()
return self._metadata
@property
def dataset_name(self):
if not self._metadata_loaded:
self.load_metadata()
if not self._crate_helper:
raise RuntimeError("ROCrate not correctly loaded")
return self._crate_helper.name
@property
def test_metadata(self):
if not self._metadata_loaded:
self.load_metadata()
return self._test_metadata
def _get_authorizations(self):
authorizations = self.authorizations.copy()
authorizations.append(None)
return authorizations
def load_metadata(self):
errors = []
# try either with authorization hedaer and without authorization
for authorization in self._get_authorizations():
try:
auth_header = authorization.as_http_header(
) if authorization else None
logger.debug(auth_header)
self._crate_helper, self._metadata, self._test_metadata = \
self.load_metadata_files(self.uri, authorization_header=auth_header)
self._metadata_loaded = True
return self._metadata, self._test_metadata
except Exception as e:
errors.append(e)
if len([
e for e in errors
if isinstance(e, lm_exceptions.NotAuthorizedException)
]) == len(errors):
raise lm_exceptions.NotAuthorizedException()
raise lm_exceptions.LifeMonitorException("ROCrate download error",
errors=errors)
@staticmethod
def extract_rocrate(roc_link, target_path=None, authorization_header=None):
with tempfile.NamedTemporaryFile(dir="/tmp") as archive_path:
zip_archive = download_url(roc_link,
target_path=archive_path.name,
authorization=authorization_header)
logger.debug("ZIP Archive: %s", zip_archive)
roc_path = target_path or Path(tempfile.mkdtemp(dir="/tmp"))
logger.info("Extracting RO Crate @ %s", roc_path)
extract_zip(archive_path, target_path=roc_path.as_posix())
return roc_path
@classmethod
def load_metadata_files(cls, roc_link, authorization_header=None):
roc_path = cls.extract_rocrate(
roc_link, authorization_header=authorization_header)
try:
roc_posix_path = roc_path.as_posix()
logger.debug(os.listdir(roc_posix_path))
crate = ROCrateHelper(roc_posix_path)
metadata_path = Path(roc_posix_path) / crate.metadata.id
with open(metadata_path, "rt") as f:
metadata = json.load(f)
# create a new Workflow instance with the loaded metadata
test_metadata = get_old_format_tests(crate)
return crate, metadata, test_metadata
finally:
shutil.rmtree(roc_path, ignore_errors=True)