class ProjectService:
"""
Services layer for Project entities.
"""
def __init__(self):
self.logger = get_logger(__name__)
self.storage_interface = StorageInterface()
def store_project(self, current_user, is_create, selected_id, **data):
"""
We want to create/update a project entity.
"""
# Validate Unique Name
new_name = data["name"]
if len(new_name) < 1:
raise ProjectServiceException("Invalid project name!")
projects_no = dao.count_projects_for_name(new_name, selected_id)
if projects_no > 0:
err = {'name': 'Please choose another name, this one is used!'}
raise formencode.Invalid("Duplicate Name Error", {},
None,
error_dict=err)
started_operations = dao.get_operation_numbers(selected_id)[1]
if started_operations > 0:
raise ProjectServiceException(
"A project can not be renamed while operations are still running!"
)
if is_create:
current_proj = Project(new_name, current_user.id,
data["description"])
self.storage_interface.get_project_folder(current_proj.name)
else:
try:
current_proj = dao.get_project_by_id(selected_id)
except Exception as excep:
self.logger.exception("An error has occurred!")
raise ProjectServiceException(str(excep))
if current_proj.name != new_name:
self.storage_interface.rename_project(current_proj.name,
new_name)
current_proj.name = new_name
current_proj.description = data["description"]
# Commit to make sure we have a valid ID
current_proj.refresh_update_date()
_, metadata_proj = current_proj.to_dict()
self.storage_interface.write_project_metadata(metadata_proj)
current_proj = dao.store_entity(current_proj)
# Retrieve, to initialize lazy attributes
current_proj = dao.get_project_by_id(current_proj.id)
# Update share settings on current Project entity
visited_pages = []
prj_admin = current_proj.administrator.username
if 'visited_pages' in data and data['visited_pages']:
visited_pages = data['visited_pages'].split(',')
for page in visited_pages:
members = UserService.retrieve_users_except([prj_admin], int(page),
MEMBERS_PAGE_SIZE)[0]
members = [m.id for m in members]
dao.delete_members_for_project(current_proj.id, members)
selected_user_ids = data["users"]
if is_create and current_user.id not in selected_user_ids:
# Make the project admin also member of the current project
selected_user_ids.append(current_user.id)
dao.add_members_to_project(current_proj.id, selected_user_ids)
# Finish operation
self.logger.debug("Edit/Save OK for project:" + str(current_proj.id) +
' by user:'******'-'
result["count"] = one_op[2]
result["gid"] = one_op[13]
operation_group_id = one_op[3]
if operation_group_id is not None and operation_group_id:
try:
operation_group = dao.get_generic_entity(
OperationGroup, operation_group_id)[0]
result["group"] = operation_group.name
result["group"] = result["group"].replace("_", " ")
result["operation_group_id"] = operation_group.id
datatype_group = dao.get_datatypegroup_by_op_group_id(
operation_group_id)
result[
"datatype_group_gid"] = datatype_group.gid if datatype_group is not None else None
result["gid"] = operation_group.gid
# Filter only viewers for current DataTypeGroup entity:
if datatype_group is None:
view_groups = None
else:
view_groups = AlgorithmService(
).get_visualizers_for_group(datatype_group.gid)
result["view_groups"] = view_groups
except Exception:
self.logger.exception(
"We will ignore group on entity:" + str(one_op))
result["datatype_group_gid"] = None
else:
result['group'] = None
result['datatype_group_gid'] = None
result["algorithm"] = dao.get_algorithm_by_id(one_op[4])
result["user"] = dao.get_user_by_id(one_op[5])
if type(one_op[6]) is str:
result["create"] = string2date(str(one_op[6]))
else:
result["create"] = one_op[6]
if type(one_op[7]) is str:
result["start"] = string2date(str(one_op[7]))
else:
result["start"] = one_op[7]
if type(one_op[8]) is str:
result["complete"] = string2date(str(one_op[8]))
else:
result["complete"] = one_op[8]
if result["complete"] is not None and result[
"start"] is not None:
result["duration"] = format_timedelta(result["complete"] -
result["start"])
result["status"] = one_op[9]
result["additional"] = one_op[10]
result["visible"] = True if one_op[11] > 0 else False
result['operation_tag'] = one_op[12]
if not result['group']:
result['results'] = dao.get_results_for_operation(
result['id'])
else:
result['results'] = None
operations.append(result)
except Exception:
# We got an exception when processing one Operation Row. We will continue with the rest of the rows.
self.logger.exception(
"Could not prepare operation for display:" + str(one_op))
return selected_project, total_ops_nr, operations, pages_no
def retrieve_projects_for_user(self, user_id, current_page=1):
"""
Return a list with all Projects visible for current user.
"""
start_idx = PROJECTS_PAGE_SIZE * (current_page - 1)
total = dao.get_projects_for_user(user_id, is_count=True)
available_projects = dao.get_projects_for_user(user_id, start_idx,
PROJECTS_PAGE_SIZE)
pages_no = total // PROJECTS_PAGE_SIZE + (1 if total %
PROJECTS_PAGE_SIZE else 0)
for prj in available_projects:
fns, sta, err, canceled, pending = dao.get_operation_numbers(
prj.id)
prj.operations_finished = fns
prj.operations_started = sta
prj.operations_error = err
prj.operations_canceled = canceled
prj.operations_pending = pending
prj.disk_size = dao.get_project_disk_size(prj.id)
prj.disk_size_human = format_bytes_human(prj.disk_size)
self.logger.debug("Displaying " + str(len(available_projects)) +
" projects in UI for user " + str(user_id))
return available_projects, pages_no
@staticmethod
def retrieve_all_user_projects(user_id,
page_start=0,
page_size=PROJECTS_PAGE_SIZE):
"""
Return a list with all projects visible for current user, without pagination.
"""
return dao.get_projects_for_user(user_id,
page_start=page_start,
page_size=page_size)
@staticmethod
def get_linkable_projects_for_user(user_id, data_id):
"""
Find projects with are visible for current user, and in which current datatype hasn't been linked yet.
"""
return dao.get_linkable_projects_for_user(user_id, data_id)
@transactional
def remove_project(self, project_id):
"""
Remove Project from DB and File Storage.
"""
try:
project2delete = dao.get_project_by_id(project_id)
self.logger.debug("Deleting project: id=" + str(project_id) +
' name=' + project2delete.name)
project_datatypes = dao.get_datatypes_in_project(project_id)
project_datatypes.sort(key=lambda dt: dt.create_date, reverse=True)
for one_data in project_datatypes:
self.remove_datatype(project_id, one_data.gid, True)
links = dao.get_links_for_project(project_id)
for one_link in links:
dao.remove_entity(Links, one_link.id)
project_bursts = dao.get_bursts_for_project(project_id)
for burst in project_bursts:
dao.remove_entity(burst.__class__, burst.id)
self.storage_interface.remove_project(project2delete)
dao.delete_project(project_id)
self.logger.debug("Deleted project: id=" + str(project_id) +
' name=' + project2delete.name)
except RemoveDataTypeException as excep:
self.logger.exception("Could not execute operation Node Remove!")
raise ProjectServiceException(str(excep))
except FileStructureException as excep:
self.logger.exception("Could not delete because of rights!")
raise ProjectServiceException(str(excep))
except Exception as excep:
self.logger.exception(str(excep))
raise ProjectServiceException(str(excep))
# ----------------- Methods for populating Data-Structure Page ---------------
@staticmethod
def get_datatype_in_group(group):
"""
Return all dataTypes that are the result of the same DTgroup.
"""
return dao.get_datatype_in_group(datatype_group_id=group)
@staticmethod
def get_datatypes_from_datatype_group(datatype_group_id):
"""
Retrieve all dataType which are part from the given dataType group.
"""
return dao.get_datatypes_from_datatype_group(datatype_group_id)
@staticmethod
def load_operation_by_gid(operation_gid):
""" Retrieve loaded Operation from DB"""
return dao.get_operation_by_gid(operation_gid)
@staticmethod
def load_operation_lazy_by_gid(operation_gid):
""" Retrieve lazy Operation from DB"""
return dao.get_operation_lazy_by_gid(operation_gid)
@staticmethod
def get_operation_group_by_id(operation_group_id):
""" Loads OperationGroup from DB"""
return dao.get_operationgroup_by_id(operation_group_id)
@staticmethod
def get_operation_group_by_gid(operation_group_gid):
""" Loads OperationGroup from DB"""
return dao.get_operationgroup_by_gid(operation_group_gid)
@staticmethod
def get_operations_in_group(operation_group):
""" Return all the operations from an operation group. """
return dao.get_operations_in_group(operation_group.id)
@staticmethod
def is_upload_operation(operation_gid):
""" Returns True only if the operation with the given GID is an upload operation. """
return dao.is_upload_operation(operation_gid)
@staticmethod
def get_all_operations_for_uploaders(project_id):
""" Returns all finished upload operations. """
return dao.get_all_operations_for_uploaders(project_id)
def set_operation_and_group_visibility(self,
entity_gid,
is_visible,
is_operation_group=False):
"""
Sets the operation visibility.
If 'is_operation_group' is True than this method will change the visibility for all
the operation from the OperationGroup with the GID field equal to 'entity_gid'.
"""
def set_visibility(op):
# workaround:
# 'reload' the operation so that it has the project property set.
# get_operations_in_group does not eager load it and now we're out of a sqlalchemy session
# write_operation_metadata requires that property
op = dao.get_operation_by_id(op.id)
# end hack
op.visible = is_visible
dao.store_entity(op)
def set_group_descendants_visibility(operation_group_id):
ops_in_group = dao.get_operations_in_group(operation_group_id)
for group_op in ops_in_group:
set_visibility(group_op)
if is_operation_group:
op_group_id = dao.get_operationgroup_by_gid(entity_gid).id
set_group_descendants_visibility(op_group_id)
else:
operation = dao.get_operation_by_gid(entity_gid)
# we assure that if the operation belongs to a group than the visibility will be changed for the entire group
if operation.fk_operation_group is not None:
set_group_descendants_visibility(operation.fk_operation_group)
else:
set_visibility(operation)
def get_operation_details(self, operation_gid, is_group):
"""
:returns: an entity OperationOverlayDetails filled with all information for current operation details.
"""
if is_group:
operation_group = self.get_operation_group_by_gid(operation_gid)
operation = dao.get_operations_in_group(operation_group.id, False,
True)
# Reload, to make sure all attributes lazy are populated as well.
operation = dao.get_operation_by_gid(operation.gid)
no_of_op_in_group = dao.get_operations_in_group(operation_group.id,
is_count=True)
datatype_group = self.get_datatypegroup_by_op_group_id(
operation_group.id)
count_result = dao.count_datatypes_in_group(datatype_group.id)
else:
operation = dao.get_operation_by_gid(operation_gid)
if operation is None:
return None
no_of_op_in_group = 1
count_result = dao.count_resulted_datatypes(operation.id)
user_display_name = dao.get_user_by_id(
operation.fk_launched_by).display_name
burst = dao.get_burst_for_operation_id(operation.id)
datatypes_param, all_special_params = self._review_operation_inputs(
operation.gid)
op_pid = dao.get_operation_process_for_operation(operation.id)
op_details = OperationOverlayDetails(operation, user_display_name,
len(datatypes_param),
count_result, burst,
no_of_op_in_group, op_pid)
# Add all parameter which are set differently by the user on this Operation.
if all_special_params is not None:
op_details.add_scientific_fields(all_special_params)
return op_details
@staticmethod
def get_filterable_meta():
"""
Contains all the attributes by which
the user can structure the tree of DataTypes
"""
return DataTypeMetaData.get_filterable_meta()
def get_project_structure(self, project, visibility_filter, first_level,
second_level, filter_value):
"""
Find all DataTypes (including the linked ones and the groups) relevant for the current project.
In case of a problem, will return an empty list.
"""
metadata_list = []
dt_list = dao.get_data_in_project(project.id, visibility_filter,
filter_value)
for dt in dt_list:
# Prepare the DT results from DB, for usage in controller, by converting into DataTypeMetaData objects
data = {}
is_group = False
group_op = None
# Filter by dt.type, otherwise Links to individual DT inside a group will be mistaken
if dt.type == "DataTypeGroup" and dt.parent_operation.operation_group is not None:
is_group = True
group_op = dt.parent_operation.operation_group
# All these fields are necessary here for dynamic Tree levels.
data[DataTypeMetaData.KEY_DATATYPE_ID] = dt.id
data[DataTypeMetaData.KEY_GID] = dt.gid
data[DataTypeMetaData.KEY_NODE_TYPE] = dt.display_type
data[DataTypeMetaData.KEY_STATE] = dt.state
data[DataTypeMetaData.KEY_SUBJECT] = str(dt.subject)
data[DataTypeMetaData.KEY_TITLE] = dt.display_name
data[DataTypeMetaData.KEY_RELEVANCY] = dt.visible
data[DataTypeMetaData.
KEY_LINK] = dt.parent_operation.fk_launched_in != project.id
data[DataTypeMetaData.
KEY_TAG_1] = dt.user_tag_1 if dt.user_tag_1 else ''
data[DataTypeMetaData.
KEY_TAG_2] = dt.user_tag_2 if dt.user_tag_2 else ''
data[DataTypeMetaData.
KEY_TAG_3] = dt.user_tag_3 if dt.user_tag_3 else ''
data[DataTypeMetaData.
KEY_TAG_4] = dt.user_tag_4 if dt.user_tag_4 else ''
data[DataTypeMetaData.
KEY_TAG_5] = dt.user_tag_5 if dt.user_tag_5 else ''
# Operation related fields:
operation_name = CommonDetails.compute_operation_name(
dt.parent_operation.algorithm.algorithm_category.displayname,
dt.parent_operation.algorithm.displayname)
data[DataTypeMetaData.KEY_OPERATION_TYPE] = operation_name
data[
DataTypeMetaData.
KEY_OPERATION_ALGORITHM] = dt.parent_operation.algorithm.displayname
data[DataTypeMetaData.
KEY_AUTHOR] = dt.parent_operation.user.username
data[
DataTypeMetaData.
KEY_OPERATION_TAG] = group_op.name if is_group else dt.parent_operation.user_group
data[DataTypeMetaData.
KEY_OP_GROUP_ID] = group_op.id if is_group else None
completion_date = dt.parent_operation.completion_date
string_year = completion_date.strftime(
MONTH_YEAR_FORMAT) if completion_date is not None else ""
string_month = completion_date.strftime(
DAY_MONTH_YEAR_FORMAT) if completion_date is not None else ""
data[DataTypeMetaData.KEY_DATE] = date2string(completion_date) if (
completion_date is not None) else ''
data[DataTypeMetaData.KEY_CREATE_DATA_MONTH] = string_year
data[DataTypeMetaData.KEY_CREATE_DATA_DAY] = string_month
data[
DataTypeMetaData.
KEY_BURST] = dt._parent_burst.name if dt._parent_burst is not None else '-None-'
metadata_list.append(DataTypeMetaData(data, dt.invalid))
return StructureNode.metadata2tree(metadata_list, first_level,
second_level, project.id,
project.name)
@staticmethod
def get_datatype_details(datatype_gid):
"""
:returns: an array. First entry in array is an instance of DataTypeOverlayDetails\
The second one contains all the possible states for the specified dataType.
"""
meta_atts = DataTypeOverlayDetails()
states = DataTypeMetaData.STATES
try:
datatype_result = dao.get_datatype_details(datatype_gid)
meta_atts.fill_from_datatype(datatype_result,
datatype_result._parent_burst)
return meta_atts, states, datatype_result
except Exception:
# We ignore exception here (it was logged above, and we want to return no details).
return meta_atts, states, None
def _remove_project_node_files(self,
project_id,
gid,
skip_validation=False):
"""
Delegate removal of a node in the structure of the project.
In case of a problem will THROW StructureException.
"""
try:
project = self.find_project(project_id)
datatype = dao.get_datatype_by_gid(gid)
links = dao.get_links_for_datatype(datatype.id)
op = dao.get_operation_by_id(datatype.fk_from_operation)
if links:
was_link = False
for link in links:
# This means it's only a link and we need to remove it
if link.fk_from_datatype == datatype.id and link.fk_to_project == project.id:
dao.remove_entity(Links, link.id)
was_link = True
if not was_link:
# Create a clone of the operation
# There is no view_model so the view_model_gid is None
new_op = Operation(
op.view_model_gid,
dao.get_system_user().id, links[0].fk_to_project,
datatype.parent_operation.fk_from_algo,
datatype.parent_operation.status,
datatype.parent_operation.start_date,
datatype.parent_operation.completion_date,
datatype.parent_operation.fk_operation_group,
datatype.parent_operation.additional_info,
datatype.parent_operation.user_group,
datatype.parent_operation.range_values)
new_op = dao.store_entity(new_op)
to_project = self.find_project(links[0].fk_to_project)
to_project_path = self.storage_interface.get_project_folder(
to_project.name)
full_path = h5.path_for_stored_index(datatype)
old_folder = self.storage_interface.get_project_folder(
project.name, str(op.id))
vm_full_path = h5.determine_filepath(
op.view_model_gid, old_folder)
self.storage_interface.move_datatype_with_sync(
to_project, to_project_path, new_op.id, full_path,
vm_full_path)
datatype.fk_from_operation = new_op.id
datatype.parent_operation = new_op
dao.store_entity(datatype)
dao.remove_entity(Links, links[0].id)
else:
specific_remover = get_remover(datatype.type)(datatype)
specific_remover.remove_datatype(skip_validation)
except RemoveDataTypeException:
self.logger.exception("Could not execute operation Node Remove!")
raise
except FileStructureException:
self.logger.exception("Remove operation failed")
raise StructureException(
"Remove operation failed for unknown reasons.Please contact system administrator."
)
def remove_operation(self, operation_id):
"""
Remove a given operation
"""
operation = dao.try_get_operation_by_id(operation_id)
if operation is not None:
self.logger.debug("Deleting operation %s " % operation)
datatypes_for_op = dao.get_results_for_operation(operation_id)
for dt in reversed(datatypes_for_op):
self.remove_datatype(operation.project.id, dt.gid, False)
# Here the Operation is mot probably already removed - in case DTs were found inside
# but we still remove it for the case when no DTs exist
dao.remove_entity(Operation, operation.id)
self.storage_interface.remove_operation_data(
operation.project.name, operation_id)
self.storage_interface.push_folder_to_sync(operation.project.name)
self.logger.debug("Finished deleting operation %s " % operation)
else:
self.logger.warning(
"Attempt to delete operation with id=%s which no longer exists."
% operation_id)
def remove_datatype(self, project_id, datatype_gid, skip_validation=False):
"""
Method used for removing a dataType. If the given dataType is a DatatypeGroup
or a dataType from a DataTypeGroup than this method will remove the entire group.
The operation(s) used for creating the dataType(s) will also be removed.
"""
datatype = dao.get_datatype_by_gid(datatype_gid)
if datatype is None:
self.logger.warning(
"Attempt to delete DT[%s] which no longer exists." %
datatype_gid)
return
is_datatype_group = False
datatype_group = None
if dao.is_datatype_group(datatype_gid):
is_datatype_group = True
datatype_group = datatype
elif datatype.fk_datatype_group is not None:
is_datatype_group = True
datatype_group = dao.get_datatype_by_id(datatype.fk_datatype_group)
operations_set = [datatype.fk_from_operation]
correct = True
if is_datatype_group:
operations_set = [datatype_group.fk_from_operation]
self.logger.debug("Removing datatype group %s" % datatype_group)
if datatype_group.fk_parent_burst:
burst = dao.get_generic_entity(BurstConfiguration,
datatype_group.fk_parent_burst,
'gid')[0]
dao.remove_entity(BurstConfiguration, burst.id)
if burst.fk_metric_operation_group:
correct = correct and self._remove_operation_group(
burst.fk_metric_operation_group, project_id,
skip_validation, operations_set)
if burst.fk_operation_group:
correct = correct and self._remove_operation_group(
burst.fk_operation_group, project_id, skip_validation,
operations_set)
else:
self._remove_datatype_group_dts(project_id, datatype_group.id,
skip_validation,
operations_set)
datatype_group = dao.get_datatype_group_by_gid(
datatype_group.gid)
dao.remove_entity(DataTypeGroup, datatype.id)
correct = correct and dao.remove_entity(
OperationGroup, datatype_group.fk_operation_group)
else:
self.logger.debug("Removing datatype %s" % datatype)
self._remove_project_node_files(project_id, datatype.gid,
skip_validation)
# Remove Operation entity in case no other DataType needs them.
project = dao.get_project_by_id(project_id)
for operation_id in operations_set:
dependent_dt = dao.get_generic_entity(DataType, operation_id,
"fk_from_operation")
if len(dependent_dt) > 0:
# Do not remove Operation in case DataType still exist referring it.
continue
op_burst = dao.get_burst_for_operation_id(operation_id)
if op_burst:
correct = correct and dao.remove_entity(
BurstConfiguration, op_burst.id)
correct = correct and dao.remove_entity(Operation, operation_id)
# Make sure Operation folder is removed
self.storage_interface.remove_operation_data(
project.name, operation_id)
self.storage_interface.push_folder_to_sync(project.name)
if not correct:
raise RemoveDataTypeException("Could not remove DataType " +
str(datatype_gid))
def _remove_operation_group(self, operation_group_id, project_id,
skip_validation, operations_set):
metrics_groups = dao.get_generic_entity(DataTypeGroup,
operation_group_id,
'fk_operation_group')
if len(metrics_groups) > 0:
metric_datatype_group_id = metrics_groups[0].id
self._remove_datatype_group_dts(project_id,
metric_datatype_group_id,
skip_validation, operations_set)
dao.remove_entity(DataTypeGroup, metric_datatype_group_id)
return dao.remove_entity(OperationGroup, operation_group_id)
def _remove_datatype_group_dts(self, project_id, dt_group_id,
skip_validation, operations_set):
data_list = dao.get_datatypes_from_datatype_group(dt_group_id)
for adata in data_list:
self._remove_project_node_files(project_id, adata.gid,
skip_validation)
if adata.fk_from_operation not in operations_set:
operations_set.append(adata.fk_from_operation)
def update_metadata(self, submit_data):
"""
Update DataType/ DataTypeGroup metadata
THROW StructureException when input data is invalid.
"""
new_data = dict()
for key in DataTypeOverlayDetails().meta_attributes_list:
if key in submit_data:
value = submit_data[key]
if value == "None":
value = None
if value == "" and key in [
CommonDetails.CODE_OPERATION_TAG,
CommonDetails.CODE_OPERATION_GROUP_ID
]:
value = None
new_data[key] = value
try:
if (CommonDetails.CODE_OPERATION_GROUP_ID in new_data
and new_data[CommonDetails.CODE_OPERATION_GROUP_ID]):
# We need to edit a group
all_data_in_group = dao.get_datatype_in_group(
operation_group_id=new_data[
CommonDetails.CODE_OPERATION_GROUP_ID])
if len(all_data_in_group) < 1:
raise StructureException(
"Inconsistent group, can not be updated!")
# datatype_group = dao.get_generic_entity(DataTypeGroup, all_data_in_group[0].fk_datatype_group)[0]
# all_data_in_group.append(datatype_group)
for datatype in all_data_in_group:
self._edit_data(datatype, new_data, True)
else:
# Get the required DataType and operation from DB to store changes that will be done in XML.
gid = new_data[CommonDetails.CODE_GID]
datatype = dao.get_datatype_by_gid(gid)
self._edit_data(datatype, new_data)
except Exception as excep:
self.logger.exception(excep)
raise StructureException(str(excep))
def _edit_data(self, datatype, new_data, from_group=False):
# type: (DataType, dict, bool) -> None
"""
Private method, used for editing a meta-data XML file and a DataType row
for a given custom DataType entity with new dictionary of data from UI.
"""
# 1. First update Operation fields:
# Update group field if possible
new_group_name = new_data[CommonDetails.CODE_OPERATION_TAG]
empty_group_value = (new_group_name is None or new_group_name == "")
if from_group:
if empty_group_value:
raise StructureException("Empty group is not allowed!")
group = dao.get_generic_entity(
OperationGroup,
new_data[CommonDetails.CODE_OPERATION_GROUP_ID])
if group and len(group) > 0 and new_group_name != group[0].name:
group = group[0]
exists_group = dao.get_generic_entity(OperationGroup,
new_group_name, 'name')
if exists_group:
raise StructureException("Group '" + new_group_name +
"' already exists.")
group.name = new_group_name
dao.store_entity(group)
else:
operation = dao.get_operation_by_id(datatype.fk_from_operation)
operation.user_group = new_group_name
dao.store_entity(operation)
op_folder = self.storage_interface.get_project_folder(
operation.project.name, str(operation.id))
vm_gid = operation.view_model_gid
view_model_file = h5.determine_filepath(vm_gid, op_folder)
if view_model_file:
view_model_class = H5File.determine_type(view_model_file)
view_model = view_model_class()
with ViewModelH5(view_model_file, view_model) as f:
ga = f.load_generic_attributes()
ga.operation_tag = new_group_name
f.store_generic_attributes(ga, False)
else:
self.logger.warning(
"Could not find ViewModel H5 file for op: {}".format(
operation))
# 2. Update GenericAttributes in the associated H5 files:
h5_path = h5.path_for_stored_index(datatype)
with H5File.from_file(h5_path) as f:
ga = f.load_generic_attributes()
ga.subject = new_data[DataTypeOverlayDetails.DATA_SUBJECT]
ga.state = new_data[DataTypeOverlayDetails.DATA_STATE]
ga.operation_tag = new_group_name
if DataTypeOverlayDetails.DATA_TAG_1 in new_data:
ga.user_tag_1 = new_data[DataTypeOverlayDetails.DATA_TAG_1]
if DataTypeOverlayDetails.DATA_TAG_2 in new_data:
ga.user_tag_2 = new_data[DataTypeOverlayDetails.DATA_TAG_2]
if DataTypeOverlayDetails.DATA_TAG_3 in new_data:
ga.user_tag_3 = new_data[DataTypeOverlayDetails.DATA_TAG_3]
if DataTypeOverlayDetails.DATA_TAG_4 in new_data:
ga.user_tag_4 = new_data[DataTypeOverlayDetails.DATA_TAG_4]
if DataTypeOverlayDetails.DATA_TAG_5 in new_data:
ga.user_tag_5 = new_data[DataTypeOverlayDetails.DATA_TAG_5]
f.store_generic_attributes(ga, False)
# 3. Update MetaData in DT Index DB as well.
datatype.fill_from_generic_attributes(ga)
dao.store_entity(datatype)
def _review_operation_inputs(self, operation_gid):
"""
:returns: A list of DataTypes that are used as input parameters for the specified operation.
And a dictionary will all operation parameters different then the default ones.
"""
operation = dao.get_operation_by_gid(operation_gid)
try:
adapter = ABCAdapter.build_adapter(operation.algorithm)
return review_operation_inputs_from_adapter(adapter, operation)
except Exception:
self.logger.exception("Could not load details for operation %s" %
operation_gid)
if operation.view_model_gid:
changed_parameters = dict(
Warning=
"Algorithm changed dramatically. We can not offer more details"
)
else:
changed_parameters = dict(
Warning=
"GID parameter is missing. Old implementation of the operation."
)
return [], changed_parameters
@staticmethod
def get_results_for_operation(operation_id, selected_filter=None):
"""
Retrieve the DataTypes entities resulted after the execution of the given operation.
"""
return dao.get_results_for_operation(operation_id, selected_filter)
@staticmethod
def get_datatype_by_id(datatype_id):
"""Retrieve a DataType DB reference by its id."""
return dao.get_datatype_by_id(datatype_id)
@staticmethod
def get_datatypegroup_by_gid(datatypegroup_gid):
""" Returns the DataTypeGroup with the specified gid. """
return dao.get_datatype_group_by_gid(datatypegroup_gid)
@staticmethod
def get_datatypegroup_by_op_group_id(operation_group_id):
""" Returns the DataTypeGroup with the specified id. """
return dao.get_datatypegroup_by_op_group_id(operation_group_id)
@staticmethod
def get_datatypes_in_project(project_id, only_visible=False):
return dao.get_data_in_project(project_id, only_visible)
@staticmethod
def set_datatype_visibility(datatype_gid, is_visible):
"""
Sets the dataType visibility. If the given dataType is a dataType group or it is part of a
dataType group than this method will set the visibility for each dataType from this group.
"""
def set_visibility(dt):
""" set visibility flag, persist in db and h5"""
dt.visible = is_visible
dt = dao.store_entity(dt)
h5_path = h5.path_for_stored_index(dt)
with H5File.from_file(h5_path) as f:
f.visible.store(is_visible)
def set_group_descendants_visibility(datatype_group_id):
datatypes_in_group = dao.get_datatypes_from_datatype_group(
datatype_group_id)
for group_dt in datatypes_in_group:
set_visibility(group_dt)
datatype = dao.get_datatype_by_gid(datatype_gid)
if isinstance(datatype, DataTypeGroup): # datatype is a group
set_group_descendants_visibility(datatype.id)
datatype.visible = is_visible
dao.store_entity(datatype)
elif datatype.fk_datatype_group is not None: # datatype is member of a group
set_group_descendants_visibility(datatype.fk_datatype_group)
# the datatype to be updated is the parent datatype group
parent = dao.get_datatype_by_id(datatype.fk_datatype_group)
parent.visible = is_visible
dao.store_entity(parent)
else:
# update the single datatype.
set_visibility(datatype)
@staticmethod
def is_datatype_group(datatype_gid):
""" Used to check if the dataType with the specified GID is a DataTypeGroup. """
return dao.is_datatype_group(datatype_gid)
def get_linked_datatypes_storage_path(self, project):
"""
:return: the file paths to the datatypes that are linked in `project`
"""
paths = []
for lnk_dt in dao.get_linked_datatypes_in_project(project.id):
# get datatype as a mapped type
path = h5.path_for_stored_index(lnk_dt)
if path is not None:
paths.append(path)
else:
self.logger.warning(
"Problem when trying to retrieve path on %s:%s!" %
(lnk_dt.type, lnk_dt.gid))
return paths
class FigureService:
"""
Service layer for Figure entities.
"""
_TYPE_PNG = "png"
_TYPE_SVG = "svg"
_BRANDING_BAR_PNG = os.path.join(os.path.dirname(__file__), "resources",
"branding_bar.png")
_BRANDING_BAR_SVG = os.path.join(os.path.dirname(__file__), "resources",
"branding_bar.svg")
_DEFAULT_SESSION_NAME = "Default"
_DEFAULT_IMAGE_FILE_NAME = "snapshot."
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.storage_interface = StorageInterface()
def _write_png(self, store_path, export_data):
img_data = base64.b64decode(export_data) # decode the image
final_image = Image.open(BytesIO(img_data)) # place it in a PIL stream
branding_bar = Image.open(
FigureService._BRANDING_BAR_PNG) # place the branding bar over
final_image.paste(branding_bar,
(0, final_image.size[1] - branding_bar.size[1]),
branding_bar)
final_image.save(store_path) # store to disk as PNG
def _write_svg(self, store_path, export_data):
dom = xml.dom.minidom.parseString(export_data)
figureSvg = dom.getElementsByTagName('svg')[
0] # get the original image
dom = xml.dom.minidom.parse(FigureService._BRANDING_BAR_SVG)
try:
width = float(figureSvg.getAttribute('width').replace('px', ''))
height = float(figureSvg.getAttribute('height').replace('px', ''))
except ValueError: # defaults when dimensions are not given
width = 1024
height = 768
figureSvg.setAttribute("width", str(width))
figureSvg.setAttribute("height", str(height))
finalSvg = dom.createElement('svg') # prepare the final svg
brandingSvg = dom.getElementsByTagName('svg')[
0] # get the branding bar
brandingSvg.setAttribute("y",
str(height)) # position it below the figure
height += float(brandingSvg.getAttribute('height').replace(
'px', '')) # increase original height with branding bar's height
finalSvg.setAttribute("width",
str(width)) # same width as original figure
finalSvg.setAttribute("height", str(height))
finalSvg.appendChild(figureSvg) # add the image
finalSvg.appendChild(brandingSvg) # and the branding bar
# Generate path where to store image
with open(store_path, 'w') as dest:
finalSvg.writexml(dest) # store to disk
def _image_path(self, project_name, img_type):
"Generate path where to store image"
images_folder = self.storage_interface.get_images_folder(project_name)
file_name = FigureService._DEFAULT_IMAGE_FILE_NAME + img_type
return utils.get_unique_file_name(images_folder, file_name)
@staticmethod
def _generate_image_name(project, user, image_name):
if not image_name:
# default to a generic name prefix
image_name = "figure"
figure_count = dao.get_figure_count(project.id, user.id) + 1
return 'TVB-%s-%s' % (image_name, figure_count)
def store_result_figure(self,
project,
user,
img_type,
export_data,
image_name=None):
"""
Store into a file, Result Image and reference in DB.
"""
store_path, file_name = self._image_path(project.name, img_type)
image_name = self._generate_image_name(project, user, image_name)
if img_type == FigureService._TYPE_PNG: # PNG file from canvas
self._write_png(store_path, export_data)
elif img_type == FigureService._TYPE_SVG: # SVG file from svg viewer
self._write_svg(store_path, export_data)
# Store entity into DB
entity = ResultFigure(user.id, project.id,
FigureService._DEFAULT_SESSION_NAME, image_name,
file_name, img_type)
entity = dao.store_entity(entity)
# Load instance from DB to have lazy fields loaded
figure = dao.load_figure(entity.id)
# Write image meta data to disk
_, meta_data = figure.to_dict()
self.storage_interface.write_image_metadata(figure, meta_data)
self.storage_interface.push_folder_to_sync(project.name)
def retrieve_result_figures(self,
project,
user,
selected_session_name='all_sessions'):
"""
Retrieve from DB all the stored Displayer previews that belongs to the specified session. The
previews are for current user and project; grouped by session.
"""
result, previews_info = dao.get_previews(project.id, user.id,
selected_session_name)
for name in result:
for figure in result[name]:
figures_folder = self.storage_interface.get_images_folder(
project.name)
figure_full_path = os.path.join(figures_folder,
figure.file_path)
# Compute the path
figure.file_path = utils.path2url_part(figure_full_path)
return result, previews_info
@staticmethod
def load_figure(figure_id):
"""
Loads a stored figure by its id.
"""
return dao.load_figure(figure_id)
def edit_result_figure(self, figure_id, **data):
"""
Retrieve and edit a previously stored figure.
"""
figure = dao.load_figure(figure_id)
figure.session_name = data['session_name']
figure.name = data['name']
dao.store_entity(figure)
# Load instance from DB to have lazy fields loaded.
figure = dao.load_figure(figure_id)
# Store figure meta data in an XML attached to the image.
_, meta_data = figure.to_dict()
self.storage_interface.write_image_metadata(figure, meta_data)
self.storage_interface.push_folder_to_sync(figure.project.name)
def remove_result_figure(self, figure_id):
"""
Remove figure from DB and file storage.
"""
figure = dao.load_figure(figure_id)
# Delete all figure related files from disk.
figures_folder = self.storage_interface.get_images_folder(
figure.project.name)
path2figure = os.path.join(figures_folder, figure.file_path)
if os.path.exists(path2figure):
os.remove(path2figure)
self.storage_interface.remove_image_metadata(figure)
self.storage_interface.push_folder_to_sync(figure.project.name)
# Remove figure reference from DB.
result = dao.remove_entity(ResultFigure, figure_id)
return result
