class ROCrate(Resource):
id = db.Column(db.Integer, db.ForeignKey(Resource.id), primary_key=True)
hosting_service_id = db.Column(db.Integer, db.ForeignKey("resource.id"), nullable=True)
hosting_service = db.relationship("Resource", uselist=False,
backref=db.backref("ro_crates", cascade="all, delete-orphan"),
foreign_keys=[hosting_service_id])
_metadata = db.Column("metadata", JSON, nullable=True)
_local_path = None
_metadata_loaded = False
__roc_helper = None
__mapper_args__ = {
'polymorphic_identity': 'ro_crate',
"inherit_condition": id == Resource.id
}
def __init__(self, uri, uuid=None, name=None,
version=None, hosting_service=None) -> None:
super().__init__(uri, uuid=uuid, name=name, version=version)
self.hosting_service = hosting_service
self.__roc_helper = None
@hybrid_property
def crate_metadata(self):
if not self._metadata_loaded:
self.load_metadata()
return self._metadata
@hybrid_property
def roc_suites(self):
return get_roc_suites(self._roc_helper)
def get_roc_suite(self, roc_suite_identifier):
try:
return self.roc_suites[roc_suite_identifier]
except KeyError:
logger.warning("Unable to find the roc_suite with identifier: %r", roc_suite_identifier)
return None
@property
def dataset_name(self):
return self._roc_helper.name
@property
def _roc_helper(self):
if not self.__roc_helper:
if not self._metadata_loaded:
self.load_metadata()
if not self.__roc_helper:
raise RuntimeError("ROCrate not correctly loaded")
return self.__roc_helper
def _get_authorizations(self):
authorizations = self.authorizations.copy()
authorizations.append(None)
return authorizations
def load_metadata(self) -> dict:
errors = []
# try either with authorization header and without authorization
for authorization in self._get_authorizations():
try:
auth_header = authorization.as_http_header() if authorization else None
logger.debug(auth_header)
self.__roc_helper, self._metadata = \
self.load_metadata_files(self.uri, authorization_header=auth_header)
self._metadata_loaded = True
return self._metadata
except lm_exceptions.NotAuthorizedException as e:
logger.info("Caught authorization error exception while downloading and processing RO-crate: %s", e)
errors.append(str(e))
raise lm_exceptions.NotAuthorizedException(detail=f"Not authorized to download {self.uri}", original_errors=errors)
@classmethod
def load_metadata_files(cls, roc_link, authorization_header=None):
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir_path = Path(tmpdir)
local_zip = download_url(roc_link,
target_path=(tmpdir_path / 'rocrate.zip').as_posix(),
authorization=authorization_header)
logger.debug("ZIP Archive: %s", local_zip)
extracted_roc_path = tmpdir_path / 'extracted-rocrate'
logger.info("Extracting RO Crate to %s", extracted_roc_path)
extract_zip(local_zip, target_path=extracted_roc_path.as_posix())
if logger.isEnabledFor(logging.DEBUG):
logger.debug("RO-crate contents: %s", ', '.join(str(s) for s in extracted_roc_path.iterdir()))
try:
crate = ROCrateHelper(extracted_roc_path.as_posix())
metadata_path = extracted_roc_path / crate.metadata.id
with open(metadata_path, "rt") as f:
metadata = json.load(f)
return crate, metadata
except Exception as e:
raise lm_exceptions.NotValidROCrateException(detail=str(e))
class WorkflowVersion(ROCrate):
id = db.Column(db.Integer, db.ForeignKey(ROCrate.id), primary_key=True)
submitter_id = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
workflow_id = \
db.Column(db.Integer, db.ForeignKey("workflow.id"), nullable=False)
workflow = db.relationship(
"Workflow",
foreign_keys=[workflow_id],
cascade="all",
backref=db.backref(
"versions",
cascade="all, delete-orphan",
collection_class=attribute_mapped_collection('version')))
test_suites = db.relationship("TestSuite",
back_populates="workflow_version",
cascade="all, delete")
submitter = db.relationship(
"User",
uselist=False,
backref=db.backref("workflows",
cascade="all, delete-orphan",
collection_class=WorkflowVersionCollection))
roc_link = association_proxy('ro_crate', 'uri')
__mapper_args__ = {'polymorphic_identity': 'workflow_version'}
def __init__(self,
workflow: Workflow,
uri,
version,
submitter: User,
uuid=None,
name=None,
hosting_service: models.WorkflowRegistry = None) -> None:
super().__init__(uri,
uuid=uuid,
name=name,
version=version,
hosting_service=hosting_service)
self.submitter = submitter
self.workflow = workflow
def __repr__(self):
return '<WorkflowVersion ({}, {}), name: {}, ro_crate link {}>'.format(
self.uuid, self.version, self.name, self.roc_link)
def check_health(self) -> dict:
health = {'healthy': True, 'issues': []}
for suite in self.test_suites:
for test_instance in suite.test_instances:
try:
testing_service = test_instance.testing_service
if not testing_service.last_test_build.is_successful():
health["healthy"] = False
except lm_exceptions.TestingServiceException as e:
health["issues"].append(str(e))
health["healthy"] = "Unknown"
return health
@hybrid_property
def authorizations(self):
auths = [a for a in self._authorizations]
if self.hosting_service and self.submitter:
for auth in self.submitter.get_authorization(self.hosting_service):
auths.append(auth)
return auths
@hybrid_property
def workflow_registry(self) -> models.WorkflowRegistry:
return self.hosting_service
@hybrid_property
def roc_link(self) -> str:
return self.uri
@property
def is_latest(self) -> bool:
return self.workflow.latest_version.version == self.version
@property
def previous_versions(self) -> List[str]:
return [
w.version for w in self.workflow.versions.values()
if w != self and w.version < self.version
]
@property
def previous_workflow_versions(self) -> List[models.WorkflowVersion]:
return [
w for w in self.workflow.versions.values()
if w != self and w.version < self.version
]
@property
def status(self) -> models.WorkflowStatus:
return models.WorkflowStatus(self)
@property
def is_healthy(self) -> Union[bool, str]:
return self.check_health()["healthy"]
def add_test_suite(self,
submitter: User,
name: str = None,
roc_suite: str = None,
definition: object = None):
return models.TestSuite(self,
submitter,
name=name,
roc_suite=roc_suite,
definition=definition)
@property
def submitter_identity(self):
# Return the submitter identity wrt the registry
identity = OAuthIdentity.find_by_user_id(self.submitter.id,
self.workflow_registry.name)
return identity.provider_user_id
def to_dict(self, test_suite=False, test_build=False, test_output=False):
health = self.check_health()
data = {
'uuid': str(self.uuid),
'version': self.version,
'name': self.name,
'roc_link': self.roc_link,
'isHealthy': health["healthy"],
'issues': health["issues"]
}
if test_suite:
data['test_suite'] = [
s.to_dict(test_build=test_build, test_output=test_output)
for s in self.test_suites
]
return data
def delete(self):
if len(self.workflow.versions) > 1:
workflow = self.workflow
self.workflow.remove_version(self)
workflow.save()
else:
self.workflow.delete()
@classmethod
def all(cls) -> List[WorkflowVersion]:
return cls.query.all()
@classmethod
def get_submitter_versions(cls, submitter: User) -> List[WorkflowVersion]:
return cls.query.filter(
WorkflowVersion.submitter_id == submitter.id).all()
@classmethod
def get_user_workflow_version(cls, owner: User, uuid,
version) -> WorkflowVersion:
try:
return cls.query\
.join(Workflow, Workflow.id == cls.workflow_id)\
.join(Permission, Permission.resource_id == cls.id)\
.filter(Workflow.uuid == lm_utils.uuid_param(uuid))\
.filter(Permission.user_id == owner.id)\
.filter(cls.version == version).one()
except NoResultFound as e:
logger.exception(e)
return None
except Exception as e:
raise lm_exceptions.LifeMonitorException(detail=str(e),
stack=str(e))
@classmethod
def get_user_workflow_versions(cls, owner: User) -> List[WorkflowVersion]:
return cls.query\
.join(Permission)\
.filter(Permission.resource_id == cls.id, Permission.user_id == owner.id).all()
@classmethod
def get_hosted_workflow_version(cls, hosting_service: Resource, uuid,
version) -> List[WorkflowVersion]:
# TODO: replace WorkflowRegistry with a more general Entity
try:
return cls.query\
.join(WorkflowRegistry, cls.hosting_service)\
.join(Workflow, Workflow.id == cls.workflow_id)\
.filter(WorkflowRegistry.uuid == lm_utils.uuid_param(hosting_service.uuid))\
.filter(Workflow.uuid == lm_utils.uuid_param(uuid))\
.filter(cls.version == version)\
.order_by(WorkflowVersion.version.desc()).one()
except NoResultFound as e:
logger.debug(e)
return None
except Exception as e:
raise lm_exceptions.LifeMonitorException(detail=str(e),
stack=str(e))
@classmethod
def get_hosted_workflow_versions(
cls, hosting_service: Resource) -> List[WorkflowVersion]:
# TODO: replace WorkflowRegistry with a more general Entity
return cls.query\
.join(WorkflowRegistry, cls.hosting_service)\
.filter(WorkflowRegistry.uuid == lm_utils.uuid_param(hosting_service.uuid))\
.order_by(WorkflowVersion.version.desc()).all()
class TestInstance(db.Model, ModelMixin):
uuid = db.Column(UUID, primary_key=True, default=_uuid.uuid4)
type = db.Column(db.String(20), nullable=False)
_test_suite_uuid = \
db.Column("test_suite_uuid", UUID, db.ForeignKey(TestSuite.uuid), nullable=False)
name = db.Column(db.String, nullable=False)
roc_instance = db.Column(db.String, nullable=True)
resource = db.Column(db.Text, nullable=False)
parameters = db.Column(JSON, nullable=True)
submitter_id = db.Column(db.Integer,
db.ForeignKey(models.User.id), nullable=True)
# configure relationships
submitter = db.relationship("User", uselist=False)
test_suite = db.relationship("TestSuite",
back_populates="test_instances",
foreign_keys=[_test_suite_uuid])
testing_service_id = db.Column(UUID, db.ForeignKey("testing_service.uuid"), nullable=False)
testing_service = db.relationship("TestingService",
foreign_keys=[testing_service_id],
backref=db.backref("test_instances", cascade="all, delete-orphan"),
uselist=False)
__mapper_args__ = {
'polymorphic_on': type,
'polymorphic_identity': 'unmanaged'
}
def __init__(self, testing_suite: TestSuite, submitter: models.User,
test_name, test_resource, testing_service: models.TestingService,
roc_instance: str = None) -> None:
self.test_suite = testing_suite
self.submitter = submitter
self.name = test_name
self.roc_instance = roc_instance
self.resource = test_resource
self.testing_service = testing_service
def __repr__(self):
return '<TestInstance {} on TestSuite {}>'.format(self.uuid, self.test_suite.uuid)
@property
def is_roc_instance(self):
return self.roc_instance is not None
@property
def managed(self):
return self.type != 'unmanaged'
@property
def last_test_build(self):
return self.testing_service.get_last_test_build(self)
def get_test_builds(self, limit=10):
return self.testing_service.get_test_builds(self, limit=limit)
def get_test_build(self, build_number):
return self.testing_service.get_test_build(self, build_number)
def to_dict(self, test_build=False, test_output=False):
data = {
'uuid': str(self.uuid),
'name': self.name,
'parameters': self.parameters,
'testing_service': self.testing_service.to_dict(test_builds=False)
}
if test_build:
data.update(self.testing_service.get_test_builds_as_dict(test_output=test_output))
return data
@classmethod
def all(cls) -> List[TestInstance]:
return cls.query.all()
@classmethod
def find_by_uuid(cls, uuid) -> TestInstance:
return cls.query.get(uuid)
class ROCrate(Resource):
id = db.Column(db.Integer, db.ForeignKey(Resource.id), primary_key=True)
hosting_service_id = db.Column(db.Integer,
db.ForeignKey("resource.id"),
nullable=True)
hosting_service = db.relationship("Resource",
uselist=False,
backref=db.backref(
"ro_crates",
cascade="all, delete-orphan"),
foreign_keys=[hosting_service_id])
_metadata = db.Column("metadata", JSON, nullable=True)
_test_metadata = None
_local_path = None
_metadata_loaded = False
__mapper_args__ = {
'polymorphic_identity': 'ro_crate',
"inherit_condition": id == Resource.id
}
def __init__(self,
uri,
uuid=None,
name=None,
version=None,
hosting_service=None) -> None:
super().__init__(uri, uuid=uuid, name=name, version=version)
self.hosting_service = hosting_service
self._crate_helper = None
@hybrid_property
def crate_metadata(self):
if not self._metadata_loaded:
self.load_metadata()
return self._metadata
@property
def dataset_name(self):
if not self._metadata_loaded:
self.load_metadata()
if not self._crate_helper:
raise RuntimeError("ROCrate not correctly loaded")
return self._crate_helper.name
@property
def test_metadata(self):
if not self._metadata_loaded:
self.load_metadata()
return self._test_metadata
def _get_authorizations(self):
authorizations = self.authorizations.copy()
authorizations.append(None)
return authorizations
def load_metadata(self):
errors = []
# try either with authorization hedaer and without authorization
for authorization in self._get_authorizations():
try:
auth_header = authorization.as_http_header(
) if authorization else None
logger.debug(auth_header)
self._crate_helper, self._metadata, self._test_metadata = \
self.load_metadata_files(self.uri, authorization_header=auth_header)
self._metadata_loaded = True
return self._metadata, self._test_metadata
except Exception as e:
errors.append(e)
if len([
e for e in errors
if isinstance(e, lm_exceptions.NotAuthorizedException)
]) == len(errors):
raise lm_exceptions.NotAuthorizedException()
raise lm_exceptions.LifeMonitorException("ROCrate download error",
errors=errors)
@staticmethod
def extract_rocrate(roc_link, target_path=None, authorization_header=None):
with tempfile.NamedTemporaryFile(dir="/tmp") as archive_path:
zip_archive = download_url(roc_link,
target_path=archive_path.name,
authorization=authorization_header)
logger.debug("ZIP Archive: %s", zip_archive)
roc_path = target_path or Path(tempfile.mkdtemp(dir="/tmp"))
logger.info("Extracting RO Crate @ %s", roc_path)
extract_zip(archive_path, target_path=roc_path.as_posix())
return roc_path
@classmethod
def load_metadata_files(cls, roc_link, authorization_header=None):
roc_path = cls.extract_rocrate(
roc_link, authorization_header=authorization_header)
try:
roc_posix_path = roc_path.as_posix()
logger.debug(os.listdir(roc_posix_path))
crate = ROCrateHelper(roc_posix_path)
metadata_path = Path(roc_posix_path) / crate.metadata.id
with open(metadata_path, "rt") as f:
metadata = json.load(f)
# create a new Workflow instance with the loaded metadata
test_metadata = get_old_format_tests(crate)
return crate, metadata, test_metadata
finally:
shutil.rmtree(roc_path, ignore_errors=True)