class ProjectService:
"""
Services layer for Project entities.
"""
def __init__(self):
self.logger = get_logger(__name__)
self.structure_helper = FilesHelper()
def store_project(self, current_user, is_create, selected_id, **data):
"""
We want to create/update a project entity.
"""
#Validate Unique Name
new_name = data["name"]
if len(new_name) < 1:
raise ProjectServiceException("Invalid project name!")
projects_no = dao.count_projects_for_name(new_name, selected_id)
if projects_no > 0:
err = {'name': 'Please choose another name, this one is used!'}
raise formencode.Invalid("Duplicate Name Error", {}, None, error_dict=err)
started_operations = dao.get_operation_numbers(selected_id)[1]
if started_operations > 0:
raise ProjectServiceException("A project can not be renamed while operations are still running!")
if is_create:
current_proj = model.Project(new_name, current_user.id, data["description"])
self.structure_helper.get_project_folder(current_proj)
else:
try:
current_proj = dao.get_project_by_id(selected_id)
except Exception, excep:
self.logger.exception("An error has occurred!")
raise ProjectServiceException(str(excep))
if current_proj.name != new_name:
self.structure_helper.rename_project_structure(current_proj.name, new_name)
current_proj.name = new_name
current_proj.description = data["description"]
#Commit to make sure we have a valid ID
current_proj.refresh_update_date()
self.structure_helper.write_project_metadata(current_proj)
current_proj = dao.store_entity(current_proj)
#Retrieve, to initialize lazy attributes
current_proj = dao.get_project_by_id(current_proj.id)
#Update share settings on current Project entity
visited_pages = []
prj_admin = current_proj.administrator.username
if 'visited_pages' in data and data['visited_pages']:
visited_pages = data['visited_pages'].split(',')
for page in visited_pages:
members = UserService.retrieve_all_users(prj_admin, int(page))[0]
members = [m.id for m in members]
dao.delete_members_for_project(current_proj.id, members)
selected_user_ids = data["users"]
dao.add_members_to_project(current_proj.id, selected_user_ids)
#Finish operation
self.logger.debug("Edit/Save OK for project:" + str(current_proj.id) + ' by user:' + current_user.username)
return current_proj
class ProjectService:
"""
Services layer for Project entities.
"""
def __init__(self):
self.logger = get_logger(__name__)
self.structure_helper = FilesHelper()
def store_project(self, current_user, is_create, selected_id, **data):
"""
We want to create/update a project entity.
"""
#Validate Unique Name
new_name = data["name"]
if len(new_name) < 1:
raise ProjectServiceException("Invalid project name!")
projects_no = dao.count_projects_for_name(new_name, selected_id)
if projects_no > 0:
err = {'name': 'Please choose another name, this one is used!'}
raise formencode.Invalid("Duplicate Name Error", {}, None, error_dict=err)
started_operations = dao.get_operation_numbers(selected_id)[1]
if started_operations > 0:
raise ProjectServiceException("A project can not be renamed while operations are still running!")
if is_create:
current_proj = model.Project(new_name, current_user.id, data["description"])
self.structure_helper.get_project_folder(current_proj)
else:
try:
current_proj = dao.get_project_by_id(selected_id)
except Exception, excep:
self.logger.exception("An error has occurred!")
raise ProjectServiceException(str(excep))
if current_proj.name != new_name:
self.structure_helper.rename_project_structure(current_proj.name, new_name)
current_proj.name = new_name
current_proj.description = data["description"]
#Commit to make sure we have a valid ID
current_proj.refresh_update_date()
self.structure_helper.write_project_metadata(current_proj)
current_proj = dao.store_entity(current_proj)
#Retrieve, to initialize lazy attributes
current_proj = dao.get_project_by_id(current_proj.id)
#Update share settings on current Project entity
visited_pages = []
prj_admin = current_proj.administrator.username
if 'visited_pages' in data and data['visited_pages']:
visited_pages = data['visited_pages'].split(',')
for page in visited_pages:
members = UserService.retrieve_all_users(prj_admin, int(page))[0]
members = [m.id for m in members]
dao.delete_members_for_project(current_proj.id, members)
selected_user_ids = data["users"]
dao.add_members_to_project(current_proj.id, selected_user_ids)
#Finish operation
self.logger.debug("Edit/Save OK for project:" + str(current_proj.id) + ' by user:' + current_user.username)
return current_proj
def export_project(self, project, optimize_size=False):
"""
Given a project root and the TVB storage_path, create a ZIP
ready for export.
:param project: project object which identifies project to be exported
"""
if project is None:
raise ExportException("Please provide project to be exported")
files_helper = FilesHelper()
project_folder = files_helper.get_project_folder(project)
project_datatypes = self._gather_project_datatypes(project, optimize_size)
to_be_exported_folders = []
considered_op_ids = []
min_dt_date = datetime.now()
if optimize_size:
## take only the DataType with visibility flag set ON
for dt in project_datatypes:
if dt[KEY_OPERATION_ID] not in considered_op_ids:
to_be_exported_folders.append({'folder': files_helper.get_project_folder(project,
str(dt[KEY_OPERATION_ID])),
'archive_path_prefix': str(dt[KEY_OPERATION_ID]) + os.sep})
considered_op_ids.append(dt[KEY_OPERATION_ID])
if min_dt_date > dt[KEY_DT_DATE]:
min_dt_date = dt[KEY_DT_DATE]
else:
to_be_exported_folders.append({'folder': project_folder,
'archive_path_prefix': '', 'exclude': ["TEMP"]})
if project_datatypes:
min_dt_date = min([dt[KEY_DT_DATE] for dt in project_datatypes])
# Compute path and name of the zip file
now = datetime.now()
date_str = now.strftime("%Y-%m-%d_%H-%M")
zip_file_name = "%s_%s.%s" % (date_str, project.name, self.ZIP_FILE_EXTENSION)
export_folder = self._build_data_export_folder(project)
result_path = os.path.join(export_folder, zip_file_name)
with TvbZip(result_path, "w") as zip_file:
# Pack project [filtered] content into a ZIP file:
LOG.debug("Done preparing, now we will write folders " + str(len(to_be_exported_folders)))
LOG.debug(str(to_be_exported_folders))
for pack in to_be_exported_folders:
zip_file.write_folder(**pack)
LOG.debug("Done exporting files, now we will write the burst configurations...")
self._export_bursts(project, project_datatypes, zip_file)
LOG.debug("Done exporting burst configurations, now we will export linked DTs")
self._export_linked_datatypes(project, zip_file, min_dt_date)
## Make sure the Project.xml file gets copied:
if optimize_size:
LOG.debug("Done linked, now we write the project xml")
zip_file.write(files_helper.get_project_meta_file_path(project.name), files_helper.TVB_PROJECT_FILE)
LOG.debug("Done, closing")
return result_path
def export_project(self, project, optimize_size=False):
"""
Given a project root and the TVB storage_path, create a ZIP
ready for export.
:param project: project object which identifies project to be exported
"""
if project is None:
raise ExportException("Please provide project to be exported")
files_helper = FilesHelper()
project_folder = files_helper.get_project_folder(project)
project_datatypes = self._gather_project_datatypes(project, optimize_size)
to_be_exported_folders = []
considered_op_ids = []
if optimize_size:
## take only the DataType with visibility flag set ON
for dt in project_datatypes:
if dt[KEY_OPERATION_ID] not in considered_op_ids:
to_be_exported_folders.append({'folder': files_helper.get_project_folder(project,
str(dt[KEY_OPERATION_ID])),
'archive_path_prefix': str(dt[KEY_OPERATION_ID]) + os.sep})
considered_op_ids.append(dt[KEY_OPERATION_ID])
else:
to_be_exported_folders.append({'folder': project_folder,
'archive_path_prefix': '', 'exclude': ["TEMP"]})
# Compute path and name of the zip file
now = datetime.now()
date_str = now.strftime("%Y-%m-%d_%H-%M")
zip_file_name = "%s_%s.%s" % (date_str, project.name, self.ZIP_FILE_EXTENSION)
export_folder = self._build_data_export_folder(project)
result_path = os.path.join(export_folder, zip_file_name)
with TvbZip(result_path, "w") as zip_file:
# Pack project [filtered] content into a ZIP file:
LOG.debug("Done preparing, now we will write folders " + str(len(to_be_exported_folders)))
LOG.debug(str(to_be_exported_folders))
for pack in to_be_exported_folders:
zip_file.write_folder(**pack)
LOG.debug("Done exporting files, now we will write the burst configurations...")
self._export_bursts(project, project_datatypes, zip_file)
LOG.debug("Done exporting burst configurations, now we will export linked DTs")
self._export_linked_datatypes(project, zip_file)
## Make sure the Project.xml file gets copied:
if optimize_size:
LOG.debug("Done linked, now we write the project xml")
zip_file.write(files_helper.get_project_meta_file_path(project.name), files_helper.TVB_PROJECT_FILE)
LOG.debug("Done, closing")
return result_path
def export(self, data, export_folder, project):
"""
Exports data type:
1. If data is a normal data type, simply exports storage file (HDF format)
2. If data is a DataTypeGroup creates a zip with all files for all data types
"""
download_file_name = self.get_export_file_name(data)
files_helper = FilesHelper()
if self.is_data_a_group(data):
all_datatypes = self._get_all_data_types_arr(data)
if all_datatypes is None or len(all_datatypes) == 0:
raise ExportException("Could not export a data type group with no data")
zip_file = os.path.join(export_folder, download_file_name)
# Now process each data type from group and add it to ZIP file
operation_folders = []
for data_type in all_datatypes:
operation_folder = files_helper.get_operation_folder(project.name, data_type.fk_from_operation)
operation_folders.append(operation_folder)
# Create ZIP archive
files_helper.zip_folders(zip_file, operation_folders, self.OPERATION_FOLDER_PREFIX)
return download_file_name, zip_file, True
else:
project_folder = files_helper.get_project_folder(project)
data_file = os.path.join(project_folder, data.get_storage_file_path())
return download_file_name, data_file, False
def index(self):
des = SerializationManager(self.simulator_context.simulator)
conn_idx = load.load_entity_by_gid(des.conf.connectivity)
model = des.conf.model
integrator = des.conf.integrator
state_vars = model.state_variables
noise_values = self.init_noise_config_values(model, integrator,
conn_idx)
initial_noise = self.group_noise_array_by_state_var(
noise_values, state_vars, conn_idx.number_of_regions)
current_project = common.get_current_project()
file_handler = FilesHelper()
conn_path = file_handler.get_project_folder(
current_project, str(conn_idx.fk_from_operation))
params = ConnectivityViewer.get_connectivity_parameters(
conn_idx, conn_path)
params.update({
'title': 'Noise configuration',
'mainContent': 'burst/noise',
'isSingleMode': True,
'submit_parameters_url': '/burst/noise/submit',
'stateVars': state_vars,
'stateVarsJson': json.dumps(state_vars),
'noiseInputValues': initial_noise[0],
'initialNoiseValues': json.dumps(initial_noise)
})
return self.fill_default_attributes(params, 'regionmodel')
def index(self):
current_user_id = common.get_logged_user().id
# In case the number of dynamics gets big we should add a filter in the ui.
dynamics = dao.get_dynamics_for_user(current_user_id)
if not dynamics:
return self.no_dynamics_page()
sim_config = common.get_from_session(common.KEY_SIMULATOR_CONFIG)
connectivity = sim_config.connectivity
if connectivity is None:
msg = 'You have to select a connectivity before setting up the region Model. '
common.set_error_message(msg)
raise ValueError(msg)
current_project = common.get_current_project()
file_handler = FilesHelper()
conn_idx = dao.get_datatype_by_gid(connectivity.hex)
conn_path = file_handler.get_project_folder(current_project, str(conn_idx.fk_from_operation))
params = ConnectivityViewer.get_connectivity_parameters(conn_idx, conn_path)
burst_config = common.get_from_session(common.KEY_BURST_CONFIG)
params.update({
'title': 'Model parameters',
'mainContent': 'burst/model_param_region',
'isSingleMode': True,
'submit_parameters_url': '/burst/modelparameters/regions/submit_model_parameters',
'dynamics': dynamics,
'dynamics_json': self._dynamics_json(dynamics),
'initial_dynamic_ids': burst_config.dynamic_ids
})
return self.fill_default_attributes(params, 'regionmodel')
class TraitUploadField(TraitField):
template = 'form_fields/upload_field.html'
def __init__(self, traited_attribute, required_type, form, name, disabled=False):
super(TraitUploadField, self).__init__(traited_attribute, form, name, disabled)
self.required_type = required_type
self.files_helper = FilesHelper()
def fill_from_post(self, post_data):
super(TraitUploadField, self).fill_from_post(post_data)
if self.data.file is None:
self.data = None
return
project = dao.get_project_by_id(self.owner.project_id)
temporary_storage = self.files_helper.get_project_folder(project, self.files_helper.TEMP_FOLDER)
file_name = None
try:
uq_name = utils.date2string(datetime.now(), True) + '_' + str(0)
file_name = TEMPORARY_PREFIX + uq_name + '_' + self.data.filename
file_name = os.path.join(temporary_storage, file_name)
with open(file_name, 'wb') as file_obj:
file_obj.write(self.data.file.read())
except Exception as excep:
# TODO: is this handled properly?
self.files_helper.remove_files([file_name])
excep.message = 'Could not continue: Invalid input files'
raise excep
if file_name:
self.data = file_name
self.owner.temporary_files.append(file_name)
class LaunchOperationResource(RestResource):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.logger = get_logger(self.__class__.__module__)
self.operation_service = OperationService()
self.project_service = ProjectService()
self.user_service = UserService()
self.files_helper = FilesHelper()
@check_permission(ProjectAccessPermission, 'project_gid')
def post(self, project_gid, algorithm_module, algorithm_classname):
"""
:generic method of launching Analyzers
"""
model_file = self.extract_file_from_request(request_file_key=RequestFileKey.LAUNCH_ANALYZERS_MODEL_FILE.value)
destination_folder = RestResource.get_destination_folder()
h5_path = RestResource.save_temporary_file(model_file, destination_folder)
try:
project = self.project_service.find_project_lazy_by_gid(project_gid)
except ProjectServiceException:
raise InvalidIdentifierException(INVALID_PROJECT_GID_MESSAGE % project_gid)
algorithm = FlowService.get_algorithm_by_module_and_class(algorithm_module, algorithm_classname)
if algorithm is None:
raise InvalidIdentifierException('No algorithm found for: %s.%s' % (algorithm_module, algorithm_classname))
try:
adapter_instance = ABCAdapter.build_adapter(algorithm)
view_model = adapter_instance.get_view_model_class()()
view_model_h5 = ViewModelH5(h5_path, view_model)
view_model_gid = view_model_h5.gid.load()
current_user = get_current_user()
operation = self.operation_service.prepare_operation(current_user.id, project.id, algorithm.id,
algorithm.algorithm_category, view_model_gid.hex, None,
{})
storage_path = self.files_helper.get_project_folder(project, str(operation.id))
if isinstance(adapter_instance, ABCUploader):
for key, value in adapter_instance.get_form_class().get_upload_information().items():
data_file = self.extract_file_from_request(request_file_key=key, file_extension=value)
data_file_path = RestResource.save_temporary_file(data_file, destination_folder)
file_name = os.path.basename(data_file_path)
upload_field = getattr(view_model_h5, key)
upload_field.store(os.path.join(storage_path, file_name))
shutil.move(data_file_path, storage_path)
shutil.move(h5_path, storage_path)
os.rmdir(destination_folder)
view_model_h5.close()
OperationService().launch_operation(operation.id, True)
except Exception as excep:
self.logger.error(excep, exc_info=True)
raise ServiceException(str(excep))
return operation.gid, HTTP_STATUS_CREATED
class TVBLoader(object):
def __init__(self, registry):
self.file_handler = FilesHelper()
self.registry = registry
def path_for_stored_index(self, dt_index_instance):
# type: (DataType) -> str
""" Given a Datatype(HasTraitsIndex) instance, build where the corresponding H5 should be or is stored"""
operation = dao.get_operation_by_id(
dt_index_instance.fk_from_operation)
operation_folder = self.file_handler.get_project_folder(
operation.project, str(operation.id))
gid = uuid.UUID(dt_index_instance.gid)
h5_file_class = self.registry.get_h5file_for_index(
dt_index_instance.__class__)
fname = '{}_{}.h5'.format(h5_file_class.file_name_base(), gid.hex)
return os.path.join(operation_folder, fname)
def path_for(self, operation_dir, h5_file_class, gid):
if isinstance(gid, str):
gid = uuid.UUID(gid)
fname = '{}_{}.h5'.format(h5_file_class.file_name_base(), gid.hex)
return os.path.join(operation_dir, fname)
def load_from_index(self, dt_index, dt_class=None):
# type: (DataType, typing.Type[HasTraits]) -> HasTraits
h5_path = self.path_for_stored_index(dt_index)
h5_file_class = self.registry.get_h5file_for_index(dt_index.__class__)
traits_class = dt_class or self.registry.get_datatype_for_index(
dt_index.__class__)
with h5_file_class(h5_path) as f:
result_dt = traits_class()
f.load_into(result_dt)
return result_dt
def load_with_references(self, file_path):
# type: (str) -> (HasTraits, GenericAttributes)
with H5File.from_file(file_path) as f:
datatype_cls = self.registry.get_datatype_for_h5file(type(f))
datatype = datatype_cls()
f.load_into(datatype)
ga = f.load_generic_attributes()
sub_dt_refs = f.gather_references()
for traited_attr, sub_gid in sub_dt_refs:
if sub_gid is None:
continue
ref_idx = dao.get_datatype_by_gid(sub_gid.hex, load_lazy=False)
ref_ht = self.load_from_index(ref_idx, traited_attr.field_type)
setattr(datatype, traited_attr.field_name, ref_ht)
return datatype, ga
def get_gifty_file_name(project_id, desired_name):
"""
Compute non-existent file name, in the TEMP folder of
the given project.
Try desired_name, and if already exists, try adding a number.
"""
if project_id:
project = dao.get_project_by_id(project_id)
file_helper = FilesHelper()
temp_path = file_helper.get_project_folder(project, FilesHelper.TEMP_FOLDER)
return get_unique_file_name(temp_path, desired_name)[0]
return get_unique_file_name(cfg.TVB_STORAGE, desired_name)[0]
def display_connectivity(connectivity_gid):
"""
Generates the html for displaying the connectivity matrix.
"""
connectivity = ABCAdapter.load_entity_by_gid(connectivity_gid)
current_project = common.get_current_project()
file_handler = FilesHelper()
conn_path = file_handler.get_project_folder(current_project, str(connectivity.fk_from_operation))
connectivity_viewer_params = ConnectivityViewer.get_connectivity_parameters(connectivity, conn_path)
template_specification = dict()
template_specification['isSingleMode'] = True
template_specification.update(connectivity_viewer_params)
return template_specification
def export_project(self, project):
"""
Given a project root and the TVB storage_path, create a ZIP
ready for export.
:param project: project object which identifies project to be exported
"""
if project is None:
raise ExportException("Please provide project to be exported")
files_helper = FilesHelper()
project_folder = files_helper.get_project_folder(project)
bursts_dict = {}
datatype_burst_mapping = {}
bursts_count = dao.get_bursts_for_project(project.id, count=True)
for start_idx in range(0, bursts_count, BURST_PAGE_SIZE):
bursts = dao.get_bursts_for_project(project.id, page_start=start_idx, page_end=start_idx + BURST_PAGE_SIZE)
for burst in bursts:
self._build_burst_export_dict(burst, bursts_dict)
datatypes_count = dao.get_datatypes_for_project(project.id, count=True)
for start_idx in range(0, datatypes_count, DATAYPES_PAGE_SIZE):
datatypes = dao.get_datatypes_for_project(project.id, page_start=start_idx,
page_end=start_idx + DATAYPES_PAGE_SIZE)
for datatype in datatypes:
datatype_burst_mapping[datatype.gid] = datatype.fk_parent_burst
# Compute path and name of the zip file
now = datetime.now()
date_str = now.strftime("%Y-%m-%d_%H-%M")
zip_file_name = "%s_%s.%s" % (date_str, project.name, self.ZIP_FILE_EXTENSION)
export_folder = self._build_data_export_folder(project)
result_path = os.path.join(export_folder, zip_file_name)
bursts_file_name = os.path.join(project_folder, BURST_INFO_FILE)
burst_info = {BURSTS_DICT_KEY: bursts_dict,
DT_BURST_MAP: datatype_burst_mapping}
with open(bursts_file_name, 'w') as bursts_file:
bursts_file.write(json.dumps(burst_info))
# pack project content into a ZIP file
result_zip = files_helper.zip_folder(result_path, project_folder)
# remove these files, since we only want them in export archive
os.remove(bursts_file_name)
return result_zip
def export(self, data, export_folder, project):
"""
Exports data type:
1. If data is a normal data type, simply exports storage file (HDF format)
2. If data is a DataTypeGroup creates a zip with all files for all data types
"""
download_file_name = self.get_export_file_name(data)
files_helper = FilesHelper()
if self.is_data_a_group(data):
all_datatypes = self._get_all_data_types_arr(data)
if all_datatypes is None or len(all_datatypes) == 0:
raise ExportException(
"Could not export a data type group with no data")
zip_file = os.path.join(export_folder, download_file_name)
# Now process each data type from group and add it to ZIP file
operation_folders = []
for data_type in all_datatypes:
operation_folder = files_helper.get_operation_folder(
project.name, data_type.fk_from_operation)
operation_folders.append(operation_folder)
# Create ZIP archive
files_helper.zip_folders(zip_file, operation_folders,
self.OPERATION_FOLDER_PREFIX)
return download_file_name, zip_file, True
else:
project_folder = files_helper.get_project_folder(project)
data_file = os.path.join(project_folder,
data.get_storage_file_path())
return download_file_name, data_file, False
class ABCAdapter(object):
"""
Root Abstract class for all TVB Adapters.
"""
# todo this constants copy is not nice
TYPE_SELECT = input_tree.TYPE_SELECT
TYPE_MULTIPLE = input_tree.TYPE_MULTIPLE
STATIC_ACCEPTED_TYPES = input_tree.STATIC_ACCEPTED_TYPES
KEY_TYPE = input_tree.KEY_TYPE
KEY_OPTIONS = input_tree.KEY_OPTIONS
KEY_ATTRIBUTES = input_tree.KEY_ATTRIBUTES
KEY_NAME = input_tree.KEY_NAME
KEY_DESCRIPTION = input_tree.KEY_DESCRIPTION
KEY_VALUE = input_tree.KEY_VALUE
KEY_LABEL = input_tree.KEY_LABEL
KEY_DEFAULT = input_tree.KEY_DEFAULT
KEY_DATATYPE = input_tree.KEY_DATATYPE
KEY_DTYPE = input_tree.KEY_DTYPE
KEY_DISABLED = input_tree.KEY_DISABLED
KEY_ALL = input_tree.KEY_ALL
KEY_CONDITION = input_tree.KEY_CONDITION
KEY_FILTERABLE = input_tree.KEY_FILTERABLE
KEY_REQUIRED = input_tree.KEY_REQUIRED
KEY_ID = input_tree.KEY_ID
KEY_UI_HIDE = input_tree.KEY_UI_HIDE
# TODO: move everything related to parameters PRE + POST into parameters_factory
KEYWORD_PARAMS = input_tree.KEYWORD_PARAMS
KEYWORD_SEPARATOR = input_tree.KEYWORD_SEPARATOR
KEYWORD_OPTION = input_tree.KEYWORD_OPTION
INTERFACE_ATTRIBUTES_ONLY = interface.INTERFACE_ATTRIBUTES_ONLY
INTERFACE_ATTRIBUTES = interface.INTERFACE_ATTRIBUTES
# Group that will be set for each adapter created by in build_adapter method
algorithm_group = None
_ui_display = 1
__metaclass__ = ABCMeta
def __init__(self):
# It will be populate with key from DataTypeMetaData
self.meta_data = {DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT}
self.file_handler = FilesHelper()
self.storage_path = '.'
# Will be populate with current running operation's identifier
self.operation_id = None
self.user_id = None
self.log = get_logger(self.__class__.__module__)
self.tree_manager = InputTreeManager()
@abstractmethod
def get_input_tree(self):
"""
Describes inputs and outputs of the launch method.
"""
@abstractmethod
def get_output(self):
"""
Describes inputs and outputs of the launch method.
"""
def configure(self, **kwargs):
"""
To be implemented in each Adapter that requires any specific configurations
before the actual launch.
"""
@abstractmethod
def get_required_memory_size(self, **kwargs):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter.
"""
@abstractmethod
def get_required_disk_size(self, **kwargs):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter in kilo-Bytes.
"""
def get_execution_time_approximation(self, **kwargs):
"""
Method should approximate based on input arguments, the time it will take for the operation
to finish (in seconds).
"""
return -1
@abstractmethod
def launch(self):
"""
To be implemented in each Adapter.
Will contain the logic of the Adapter.
Any returned DataType will be stored in DB, by the Framework.
"""
def add_operation_additional_info(self, message):
"""
Adds additional info on the operation to be displayed in the UI. Usually a warning message.
"""
current_op = dao.get_operation_by_id(self.operation_id)
current_op.additional_info = message
dao.store_entity(current_op)
@nan_not_allowed()
def _prelaunch(self, operation, uid=None, available_disk_space=0, **kwargs):
"""
Method to wrap LAUNCH.
Will prepare data, and store results on return.
"""
self.meta_data.update(json.loads(operation.meta_data))
self.storage_path = self.file_handler.get_project_folder(operation.project, str(operation.id))
self.operation_id = operation.id
self.current_project_id = operation.project.id
self.user_id = operation.fk_launched_by
self.configure(**kwargs)
# Compare the amount of memory the current algorithms states it needs,
# with the average between the RAM available on the OS and the free memory at the current moment.
# We do not consider only the free memory, because some OSs are freeing late and on-demand only.
total_free_memory = psutil.virtual_memory().free + psutil.swap_memory().free
total_existent_memory = psutil.virtual_memory().total + psutil.swap_memory().total
memory_reference = (total_free_memory + total_existent_memory) / 2
adapter_required_memory = self.get_required_memory_size(**kwargs)
if adapter_required_memory > memory_reference:
msg = "Machine does not have enough RAM memory for the operation (expected %.2g GB, but found %.2g GB)."
raise NoMemoryAvailableException(msg % (adapter_required_memory / 2 ** 30, memory_reference / 2 ** 30))
# Compare the expected size of the operation results with the HDD space currently available for the user
# TVB defines a quota per user.
required_disk_space = self.get_required_disk_size(**kwargs)
if available_disk_space < 0:
msg = "You have exceeded you HDD space quota by %.2f MB Stopping execution."
raise NoMemoryAvailableException(msg % (- available_disk_space / 2 ** 10))
if available_disk_space < required_disk_space:
msg = ("You only have %.2f GB of disk space available but the operation you "
"launched might require %.2f Stopping execution...")
raise NoMemoryAvailableException(msg % (available_disk_space / 2 ** 20, required_disk_space / 2 ** 20))
operation.start_now()
operation.estimated_disk_size = required_disk_space
dao.store_entity(operation)
result = self.launch(**kwargs)
if not isinstance(result, (list, tuple)):
result = [result, ]
self.__check_integrity(result)
return self._capture_operation_results(result, uid)
def _capture_operation_results(self, result, user_tag=None):
"""
After an operation was finished, make sure the results are stored
in DB storage and the correct meta-data,IDs are set.
"""
results_to_store = []
data_type_group_id = None
operation = dao.get_operation_by_id(self.operation_id)
if operation.user_group is None or len(operation.user_group) == 0:
operation.user_group = date2string(datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT)
operation = dao.store_entity(operation)
if self._is_group_launch():
data_type_group_id = dao.get_datatypegroup_by_op_group_id(operation.fk_operation_group).id
# All entities will have the same subject and state
subject = self.meta_data[DataTypeMetaData.KEY_SUBJECT]
state = self.meta_data[DataTypeMetaData.KEY_STATE]
burst_reference = None
if DataTypeMetaData.KEY_BURST in self.meta_data:
burst_reference = self.meta_data[DataTypeMetaData.KEY_BURST]
perpetuated_identifier = None
if DataTypeMetaData.KEY_TAG_1 in self.meta_data:
perpetuated_identifier = self.meta_data[DataTypeMetaData.KEY_TAG_1]
for res in result:
if res is None:
continue
res.subject = str(subject)
res.state = state
res.fk_parent_burst = burst_reference
res.fk_from_operation = self.operation_id
res.framework_metadata = self.meta_data
if not res.user_tag_1:
res.user_tag_1 = user_tag if user_tag is not None else perpetuated_identifier
else:
res.user_tag_2 = user_tag if user_tag is not None else perpetuated_identifier
res.fk_datatype_group = data_type_group_id
## Compute size-on disk, in case file-storage is used
if hasattr(res, 'storage_path') and hasattr(res, 'get_storage_file_name'):
associated_file = os.path.join(res.storage_path, res.get_storage_file_name())
res.close_file()
res.disk_size = self.file_handler.compute_size_on_disk(associated_file)
res = dao.store_entity(res)
# Write metaData
res.persist_full_metadata()
results_to_store.append(res)
del result[0:len(result)]
result.extend(results_to_store)
if len(result) and self._is_group_launch():
## Update the operation group name
operation_group = dao.get_operationgroup_by_id(operation.fk_operation_group)
operation_group.fill_operationgroup_name(result[0].type)
dao.store_entity(operation_group)
return 'Operation ' + str(self.operation_id) + ' has finished.', len(results_to_store)
def __check_integrity(self, result):
"""
Check that the returned parameters for LAUNCH operation
are of the type specified in the adapter's interface.
"""
entity_id = self.__module__ + '.' + self.__class__.__name__
for result_entity in result:
if type(result_entity) == list and len(result_entity) > 0:
#### Determine the first element not None
first_item = None
for res in result_entity:
if res is not None:
first_item = res
break
if first_item is None:
return
#### All list items are None
#### Now check if the first item has a supported type
if not self.__is_data_in_supported_types(first_item):
msg = "Unexpected DataType %s"
raise Exception(msg % type(first_item))
first_item_type = type(first_item)
for res in result_entity:
if not isinstance(res, first_item_type):
msg = '%s-Heterogeneous types (%s).Expected %s list.'
raise Exception(msg % (entity_id, type(res), first_item_type))
else:
if not self.__is_data_in_supported_types(result_entity):
msg = "Unexpected DataType %s"
raise Exception(msg % type(result_entity))
def __is_data_in_supported_types(self, data):
"""
This method checks if the provided data is one of the adapter supported return types
"""
if data is None:
return True
for supported_type in self.get_output():
if isinstance(data, supported_type):
return True
##### Data can't be mapped on any supported type !!
return False
def _is_group_launch(self):
"""
Return true if this adapter is launched from a group of operations
"""
operation = dao.get_operation_by_id(self.operation_id)
return operation.fk_operation_group is not None
@staticmethod
def load_entity_by_gid(data_gid):
"""
Load a generic DataType, specified by GID.
"""
return load_entity_by_gid(data_gid)
@staticmethod
def prepare_adapter(adapter_class):
"""
Having a subclass of ABCAdapter, prepare an instance for launching an operation with it.
"""
try:
if not issubclass(adapter_class, ABCAdapter):
raise IntrospectionException("Invalid data type: It should extend adapters.ABCAdapter!")
algo_group = dao.find_group(adapter_class.__module__, adapter_class.__name__)
adapter_instance = adapter_class()
adapter_instance.algorithm_group = algo_group
return adapter_instance
except Exception, excep:
get_logger("ABCAdapter").exception(excep)
raise IntrospectionException(str(excep))
class SimulatorService(object):
MAX_BURSTS_DISPLAYED = 50
LAUNCH_NEW = 'new'
LAUNCH_BRANCH = 'branch'
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.operation_service = OperationService()
self.files_helper = FilesHelper()
@staticmethod
def serialize_simulator(simulator, simulator_gid, simulation_state_gid, storage_path):
simulator_path = h5.path_for(storage_path, SimulatorH5, simulator_gid)
with SimulatorH5(simulator_path) as simulator_h5:
simulator_h5.gid.store(uuid.UUID(simulator_gid))
simulator_h5.store(simulator)
simulator_h5.connectivity.store(simulator.connectivity.gid)
if simulator.stimulus:
simulator_h5.stimulus.store(uuid.UUID(simulator.stimulus.gid))
if simulation_state_gid:
simulator_h5.simulation_state.store(uuid.UUID(simulation_state_gid))
return simulator_gid
@staticmethod
def deserialize_simulator(simulator_gid, storage_path):
simulator_in_path = h5.path_for(storage_path, SimulatorH5, simulator_gid)
simulator_in = Simulator()
with SimulatorH5(simulator_in_path) as simulator_in_h5:
simulator_in_h5.load_into(simulator_in)
connectivity_gid = simulator_in_h5.connectivity.load()
stimulus_gid = simulator_in_h5.stimulus.load()
simulation_state_gid = simulator_in_h5.simulation_state.load()
conn_index = dao.get_datatype_by_gid(connectivity_gid.hex)
conn = h5.load_from_index(conn_index)
simulator_in.connectivity = conn
if simulator_in.surface:
cortex_path = h5.path_for(storage_path, CortexH5, simulator_in.surface.gid)
with CortexH5(cortex_path) as cortex_h5:
local_conn_gid = cortex_h5.local_connectivity.load()
region_mapping_gid = cortex_h5.region_mapping_data.load()
region_mapping_index = dao.get_datatype_by_gid(region_mapping_gid.hex)
region_mapping_path = h5.path_for_stored_index(region_mapping_index)
region_mapping = RegionMapping()
with RegionMappingH5(region_mapping_path) as region_mapping_h5:
region_mapping_h5.load_into(region_mapping)
region_mapping.gid = region_mapping_h5.gid.load()
surf_gid = region_mapping_h5.surface.load()
surf_index = dao.get_datatype_by_gid(surf_gid.hex)
surf_h5 = h5.h5_file_for_index(surf_index)
surf = CorticalSurface()
surf_h5.load_into(surf)
surf_h5.close()
region_mapping.surface = surf
simulator_in.surface.region_mapping_data = region_mapping
if local_conn_gid:
local_conn_index = dao.get_datatype_by_gid(local_conn_gid.hex)
local_conn = h5.load_from_index(local_conn_index)
simulator_in.surface.local_connectivity = local_conn
if stimulus_gid:
stimulus_index = dao.get_datatype_by_gid(stimulus_gid.hex)
stimulus = h5.load_from_index(stimulus_index)
simulator_in.stimulus = stimulus
return simulator_in, connectivity_gid, simulation_state_gid
@transactional
def _prepare_operation(self, project_id, user_id, simulator_id, simulator_index, algo_category, op_group, metadata,
ranges=None):
operation_parameters = json.dumps({'simulator_gid': simulator_index.gid})
metadata, user_group = self.operation_service._prepare_metadata(metadata, algo_category, op_group, {})
meta_str = json.dumps(metadata)
op_group_id = None
if op_group:
op_group_id = op_group.id
operation = Operation(user_id, project_id, simulator_id, operation_parameters, op_group_id=op_group_id,
meta=meta_str, range_values=ranges)
self.logger.debug("Saving Operation(userId=" + str(user_id) + ",projectId=" + str(project_id) + "," +
str(metadata) + ",algorithmId=" + str(simulator_id) + ", ops_group= " + str(
op_group_id) + ")")
# visible_operation = visible and category.display is False
operation = dao.store_entity(operation)
# operation.visible = visible_operation
# TODO: prepare portlets/handle operation groups/no workflows
return operation
@staticmethod
def _set_simulator_range_parameter(simulator, range_parameter_name, range_parameter_value):
range_param_name_list = range_parameter_name.split('.')
current_attr = simulator
for param_name in range_param_name_list[:len(range_param_name_list) - 1]:
current_attr = getattr(current_attr, param_name)
setattr(current_attr, range_param_name_list[-1], range_parameter_value)
def async_launch_and_prepare_simulation(self, burst_config, user, project, simulator_algo,
session_stored_simulator, simulation_state_gid):
try:
simulator_index = SimulatorIndex()
metadata = {}
if burst_config:
simulator_index.fk_parent_burst = burst_config.id
metadata.update({DataTypeMetaData.KEY_BURST: burst_config.id})
dao.store_entity(simulator_index)
simulator_id = simulator_algo.id
algo_category = simulator_algo.algorithm_category
operation = self._prepare_operation(project.id, user.id, simulator_id, simulator_index, algo_category, None,
metadata)
simulator_index.fk_from_operation = operation.id
dao.store_entity(simulator_index)
storage_path = self.files_helper.get_project_folder(project, str(operation.id))
self.serialize_simulator(session_stored_simulator, simulator_index.gid, simulation_state_gid, storage_path)
wf_errs = 0
try:
OperationService().launch_operation(operation.id, True)
return operation
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
if burst_config:
BurstService2().mark_burst_finished(burst_config, error_message=str(excep))
self.logger.debug("Finished launching workflow. The operation was launched successfully, " +
str(wf_errs) + " had error on pre-launch steps")
except Exception as excep:
self.logger.error(excep)
if burst_config:
BurstService2().mark_burst_finished(burst_config, error_message=str(excep))
def async_launch_and_prepare_pse(self, burst_config, user, project, simulator_algo, range_param1, range_param2,
session_stored_simulator):
try:
simulator_id = simulator_algo.id
algo_category = simulator_algo.algorithm_category
operation_group = burst_config.operation_group
metric_operation_group = burst_config.metric_operation_group
operations = []
range_param2_values = []
if range_param2:
range_param2_values = range_param2.get_range_values()
for param1_value in range_param1.get_range_values():
for param2_value in range_param2_values:
simulator = copy.deepcopy(session_stored_simulator)
self._set_simulator_range_parameter(simulator, range_param1.name, param1_value)
self._set_simulator_range_parameter(simulator, range_param2.name, param2_value)
simulator_index = SimulatorIndex()
simulator_index.fk_parent_burst = burst_config.id
simulator_index = dao.store_entity(simulator_index)
ranges = json.dumps({range_param1.name: param1_value[0], range_param2.name: param2_value[0]})
operation = self._prepare_operation(project.id, user.id, simulator_id, simulator_index,
algo_category, operation_group,
{DataTypeMetaData.KEY_BURST: burst_config.id}, ranges)
simulator_index.fk_from_operation = operation.id
dao.store_entity(simulator_index)
storage_path = self.files_helper.get_project_folder(project, str(operation.id))
self.serialize_simulator(simulator, simulator_index.gid, None, storage_path)
operations.append(operation)
first_operation = operations[0]
datatype_group = DataTypeGroup(operation_group, operation_id=first_operation.id,
fk_parent_burst=burst_config.id,
state=json.loads(first_operation.meta_data)[DataTypeMetaData.KEY_STATE])
dao.store_entity(datatype_group)
metrics_datatype_group = DataTypeGroup(metric_operation_group, fk_parent_burst=burst_config.id)
dao.store_entity(metrics_datatype_group)
wf_errs = 0
for operation in operations:
try:
OperationService().launch_operation(operation.id, True)
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
BurstService2().mark_burst_finished(burst_config, error_message=str(excep))
self.logger.debug("Finished launching workflows. " + str(len(operations) - wf_errs) +
" were launched successfully, " + str(wf_errs) + " had error on pre-launch steps")
except Exception as excep:
self.logger.error(excep)
BurstService2().mark_burst_finished(burst_config, error_message=str(excep))
class TestFilesHelper(TransactionalTestCase):
"""
This class contains tests for the tvb.core.entities.file.files_helper module.
"""
PROJECT_NAME = "test_proj"
def transactional_setup_method(self):
"""
Set up the context needed by the tests.
"""
self.files_helper = FilesHelper()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user, self.PROJECT_NAME)
def transactional_teardown_method(self):
""" Remove generated project during tests. """
self.delete_project_folders()
def test_check_created(self):
""" Test standard flows for check created. """
self.files_helper.check_created()
assert os.path.exists(root_storage), "Storage not created!"
self.files_helper.check_created(os.path.join(root_storage, "test"))
assert os.path.exists(root_storage), "Storage not created!"
assert os.path.exists(os.path.join(root_storage, "test")), "Test directory not created!"
def test_get_project_folder(self):
"""
Test the get_project_folder method which should create a folder in case
it doesn't already exist.
"""
project_path = self.files_helper.get_project_folder(self.test_project)
assert os.path.exists(project_path), "Folder doesn't exist"
folder_path = self.files_helper.get_project_folder(self.test_project, "43")
assert os.path.exists(project_path), "Folder doesn't exist"
assert os.path.exists(folder_path), "Folder doesn't exist"
def test_rename_project_structure(self):
""" Try to rename the folder structure of a project. Standard flow. """
self.files_helper.get_project_folder(self.test_project)
path, name = self.files_helper.rename_project_structure(self.test_project.name, "new_name")
assert path != name, "Rename didn't take effect."
def test_rename_structure_same_name(self):
""" Try to rename the folder structure of a project. Same name. """
self.files_helper.get_project_folder(self.test_project)
with pytest.raises(FileStructureException):
self.files_helper.rename_project_structure(self.test_project.name, self.PROJECT_NAME)
def test_remove_project_structure(self):
""" Check that remove project structure deletes the corresponding folder. Standard flow. """
full_path = self.files_helper.get_project_folder(self.test_project)
assert os.path.exists(full_path), "Folder was not created."
self.files_helper.remove_project_structure(self.test_project.name)
assert not os.path.exists(full_path), "Project folder not deleted."
def test_write_project_metadata(self):
""" Write XML for test-project. """
self.files_helper.write_project_metadata(self.test_project)
expected_file = self.files_helper.get_project_meta_file_path(self.PROJECT_NAME)
assert os.path.exists(expected_file)
project_meta = XMLReader(expected_file).read_metadata()
loaded_project = model_project.Project(None, None)
loaded_project.from_dict(project_meta, self.test_user.id)
assert self.test_project.name == loaded_project.name
assert self.test_project.description == loaded_project.description
assert self.test_project.gid == loaded_project.gid
expected_dict = self.test_project.to_dict()[1]
del expected_dict['last_updated']
found_dict = loaded_project.to_dict()[1]
del found_dict['last_updated']
self._dictContainsSubset(expected_dict, found_dict)
self._dictContainsSubset(found_dict, expected_dict)
def test_write_operation_metadata(self):
"""
Test that a correct XML is created for an operation.
"""
operation = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)
expected_file = self.files_helper.get_operation_meta_file_path(self.PROJECT_NAME, operation.id)
assert not os.path.exists(expected_file)
self.files_helper.write_operation_metadata(operation)
assert os.path.exists(expected_file)
operation_meta = XMLReader(expected_file).read_metadata()
loaded_operation = model_operation.Operation(None, None, None, None)
loaded_operation.from_dict(operation_meta, dao, user_id=self.test_user.id)
expected_dict = operation.to_dict()[1]
found_dict = loaded_operation.to_dict()[1]
for key, value in expected_dict.items():
assert str(value) == str(found_dict[key])
# Now validate that operation metaData can be also updated
assert "new_group_name" != found_dict['user_group']
self.files_helper.update_operation_metadata(self.PROJECT_NAME, "new_group_name", operation.id)
found_dict = XMLReader(expected_file).read_metadata()
assert "new_group_name" == found_dict['user_group']
def test_remove_dt_happy_flow(self, dummy_datatype_index_factory):
"""
Happy flow for removing a file related to a DataType.
"""
datatype = dummy_datatype_index_factory()
h5_path = h5.path_for_stored_index(datatype)
assert os.path.exists(h5_path), "Test file was not created!"
self.files_helper.remove_datatype_file(h5_path)
assert not os.path.exists(h5_path), "Test file was not deleted!"
def test_remove_dt_non_existent(self, dummy_datatype_index_factory):
"""
Try to call remove on a dataType with no H5 file.
Should work.
"""
datatype = dummy_datatype_index_factory()
h5_path = h5.path_for_stored_index(datatype)
wrong_path = os.path.join(h5_path, "WRONG_PATH")
assert not os.path.exists(wrong_path)
self.files_helper.remove_datatype_file(wrong_path)
def test_move_datatype(self, dummy_datatype_index_factory):
"""
Make sure associated H5 file is moved to a correct new location.
"""
datatype = dummy_datatype_index_factory(project=self.test_project)
old_file_path = h5.path_for_stored_index(datatype)
assert os.path.exists(old_file_path), "Test file was not created!"
full_path = h5.path_for_stored_index(datatype)
self.files_helper.move_datatype(datatype, self.PROJECT_NAME + '2', "1", full_path)
assert not os.path.exists(old_file_path), "Test file was not moved!"
datatype.fk_from_operation = 43
new_file_path = os.path.join(self.files_helper.get_project_folder(self.PROJECT_NAME + '2', "1"), old_file_path.split("\\")[-1])
assert os.path.exists(new_file_path), "Test file was not created!"
def test_find_relative_path(self):
"""
Tests that relative path is computed properly.
"""
rel_path = self.files_helper.find_relative_path("/root/up/to/here/test/it/now", "/root/up/to/here")
assert rel_path == os.sep.join(["test", "it", "now"]), "Did not extract relative path as expected."
def test_remove_files_valid(self):
"""
Pass a valid list of files and check they are all removed.
"""
file_list = ["test1", "test2", "test3"]
for file_n in file_list:
fp = open(file_n, 'w')
fp.write('test')
fp.close()
for file_n in file_list:
assert os.path.isfile(file_n)
self.files_helper.remove_files(file_list)
for file_n in file_list:
assert not os.path.isfile(file_n)
def test_remove_folder(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
os.mkdir(folder_name)
assert os.path.isdir(folder_name), "Folder should be created."
self.files_helper.remove_folder(folder_name)
assert not os.path.isdir(folder_name), "Folder should be deleted."
def test_remove_folder_non_existing_ignore_exc(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
assert not os.path.isdir(folder_name), "Folder should not exist before call."
self.files_helper.remove_folder(folder_name, ignore_errors=True)
def test_remove_folder_non_existing(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
assert not os.path.isdir(folder_name), "Folder should not exist before call."
with pytest.raises(FileStructureException):
self.files_helper.remove_folder(folder_name, False)
def _dictContainsSubset(self, expected, actual, msg=None):
"""Checks whether actual is a superset of expected."""
missing = []
mismatched = []
for key, value in expected.items():
if key not in actual:
return False
elif value != actual[key]:
return False
return True
class BurstService(object):
LAUNCH_NEW = 'new'
LAUNCH_BRANCH = 'branch'
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.file_helper = FilesHelper()
def mark_burst_finished(self, burst_entity, burst_status=None, error_message=None):
"""
Mark Burst status field.
Also compute 'weight' for current burst: no of operations inside, estimate time on disk...
:param burst_entity: BurstConfiguration to be updated, at finish time.
:param burst_status: BurstConfiguration status. By default BURST_FINISHED
:param error_message: If given, set the status to error and perpetuate the message.
"""
if burst_status is None:
burst_status = BurstConfiguration.BURST_FINISHED
if error_message is not None:
burst_status = BurstConfiguration.BURST_ERROR
try:
# If there are any DataType Groups in current Burst, update their counter.
burst_dt_groups = dao.get_generic_entity(DataTypeGroup, burst_entity.id, "fk_parent_burst")
for dt_group in burst_dt_groups:
dt_group.count_results = dao.count_datatypes_in_group(dt_group.id)
dt_group.disk_size, dt_group.subject = dao.get_summary_for_group(dt_group.id)
dao.store_entity(dt_group)
# Update actual Burst entity fields
burst_entity.datatypes_number = dao.count_datatypes_in_burst(burst_entity.id)
burst_entity.status = burst_status
burst_entity.error_message = error_message
burst_entity.finish_time = datetime.now()
dao.store_entity(burst_entity)
self.update_burst_configuration_h5(burst_entity)
except Exception:
self.logger.exception("Could not correctly update Burst status and meta-data!")
burst_entity.status = burst_status
burst_entity.error_message = "Error when updating Burst Status"
burst_entity.finish_time = datetime.now()
dao.store_entity(burst_entity)
self.update_burst_configuration_h5(burst_entity)
def persist_operation_state(self, operation, operation_status, message=None):
"""
Update Operation instance state. Store it in DB and on HDD/
:param operation: Operation instance
:param operation_status: new status
:param message: message in case of error
:return: operation instance changed
"""
operation.mark_complete(operation_status, message)
operation = dao.store_entity(operation)
# update burst also
burst_config = self.get_burst_for_operation_id(operation.id)
if burst_config is not None:
burst_status = STATUS_FOR_OPERATION.get(operation_status)
self.mark_burst_finished(burst_config, burst_status, message)
return operation
@staticmethod
def get_burst_for_operation_id(operation_id):
return dao.get_burst_for_operation_id(operation_id)
def rename_burst(self, burst_id, new_name):
"""
Rename the burst given by burst_id, setting it's new name to
burst_name.
"""
burst = dao.get_burst_by_id(burst_id)
burst.name = new_name
dao.store_entity(burst)
self.update_burst_configuration_h5(burst)
@staticmethod
def get_available_bursts(project_id):
"""
Return all the burst for the current project.
"""
bursts = dao.get_bursts_for_project(project_id, page_size=MAX_BURSTS_DISPLAYED) or []
# for burst in bursts:
# burst.prepare_after_load()
return bursts
@staticmethod
def populate_burst_disk_usage(bursts):
"""
Adds a disk_usage field to each burst object.
The disk usage is computed as the sum of the datatypes generated by a burst
"""
sizes = dao.compute_bursts_disk_size([b.id for b in bursts])
for b in bursts:
b.disk_size = format_bytes_human(sizes[b.id])
def update_history_status(self, id_list):
"""
For each burst_id received in the id_list read new status from DB and return a list [id, new_status] pair.
"""
result = []
for b_id in id_list:
burst = dao.get_burst_by_id(b_id)
# burst.prepare_after_load()
if burst is not None:
if burst.status == burst.BURST_RUNNING:
running_time = datetime.now() - burst.start_time
else:
running_time = burst.finish_time - burst.start_time
running_time = format_timedelta(running_time, most_significant2=False)
if burst.status == burst.BURST_ERROR:
msg = 'Check Operations page for error Message'
else:
msg = ''
result.append([burst.id, burst.status, burst.is_group, msg, running_time])
else:
self.logger.debug("Could not find burst with id=" + str(b_id) + ". Might have been deleted by user!!")
return result
@staticmethod
def update_simulation_fields(burst_id, op_simulation_id, simulation_gid):
burst = dao.get_burst_by_id(burst_id)
burst.fk_simulation = op_simulation_id
burst.simulator_gid = simulation_gid.hex
burst = dao.store_entity(burst)
return burst
def update_burst_configuration_h5(self, burst_configuration):
# type: (BurstConfiguration) -> None
project = dao.get_project_by_id(burst_configuration.fk_project)
storage_path = self.file_helper.get_project_folder(project, str(burst_configuration.fk_simulation))
self.store_burst_configuration(burst_configuration, storage_path)
def load_burst_configuration(self, burst_config_id):
# type: (int) -> BurstConfiguration
burst_config = dao.get_burst_by_id(burst_config_id)
return burst_config
def prepare_burst_for_pse(self, burst_config):
# type: (BurstConfiguration) -> None
if burst_config.range2:
ranges = [burst_config.range1, burst_config.range2]
else:
ranges = [burst_config.range1]
operation_group = OperationGroup(burst_config.fk_project, ranges=ranges)
operation_group = dao.store_entity(operation_group)
metric_operation_group = OperationGroup(burst_config.fk_project, ranges=ranges)
metric_operation_group = dao.store_entity(metric_operation_group)
burst_config.operation_group = operation_group
burst_config.fk_operation_group = operation_group.id
burst_config.metric_operation_group = metric_operation_group
burst_config.fk_metric_operation_group = metric_operation_group.id
return dao.store_entity(burst_config)
def store_burst_configuration(self, burst_config, storage_path):
bc_path = h5.path_for(storage_path, BurstConfigurationH5, burst_config.gid)
with BurstConfigurationH5(bc_path) as bc_h5:
bc_h5.store(burst_config)
def load_burst_configuration_from_folder(self, simulator_folder, project):
bc_h5_filename = DirLoader(simulator_folder, None).find_file_for_has_traits_type(BurstConfiguration)
burst_config = BurstConfiguration(project.id)
with BurstConfigurationH5(os.path.join(simulator_folder, bc_h5_filename)) as bc_h5:
bc_h5.load_into(burst_config)
return burst_config
@staticmethod
def prepare_name(burst, project_id):
simulation_number = dao.get_number_of_bursts(project_id) + 1
if burst.name is None:
default_simulation_name = 'simulation_' + str(simulation_number)
else:
default_simulation_name = burst.name
return default_simulation_name, simulation_number
class DatatypesFactory():
"""
This class provides a set of methods that helps user to create
different data types for testing.
These data types will be automatically stored in DB and file system if needed.
"""
USER_FULL_NAME = "Datatype Factory User"
DATATYPE_STATE = "RAW_DATA"
DATATYPE_DATA = ["test", "for", "datatypes", "factory"]
DATATYPE_MEASURE_METRIC = {'v': 3}
RANGE_1 = ["row1", [1, 2, 3]]
RANGE_2 = ["row2", [0.1, 0.3, 0.5]]
user = None
project = None
operation = None
def __init__(self):
micro_postfix = "_%d" % int(time.time() * 1000000)
# Here create all structures needed later for data types creation
self.files_helper = FilesHelper()
# First create user
user = model.User("datatype_factory_user" + micro_postfix, "test_pass",
"*****@*****.**" + micro_postfix, True, "user")
self.user = dao.store_entity(user)
# Now create a project
project_service = ProjectService()
data = dict(name='DatatypesFactoryProject' + micro_postfix, description='test_desc', users=[])
self.project = project_service.store_project(self.user, True, None, **data)
# Create algorithm
alg_category = model.AlgorithmCategory('one', True)
dao.store_entity(alg_category)
alg_group = model.AlgorithmGroup("test_module1", "classname1", alg_category.id)
dao.store_entity(alg_group)
algorithm = model.Algorithm(alg_group.id, 'id', name='', req_data='', param_name='', output='')
self.algorithm = dao.store_entity(algorithm)
#Create an operation
self.meta = {DataTypeMetaData.KEY_SUBJECT: self.USER_FULL_NAME,
DataTypeMetaData.KEY_STATE: self.DATATYPE_STATE}
operation = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters',
meta=json.dumps(self.meta), status=model.STATUS_FINISHED,
method_name=ABCAdapter.LAUNCH_METHOD)
self.operation = dao.store_entity(operation)
def get_project(self):
"""
Return project to which generated data types are assigned
"""
return self.project
def get_operation(self):
"""
Return operation to which generated data types are assigned
"""
return self.operation
def get_user(self):
"""
Return user to which generated data types are assigned
"""
return self.user
def _store_datatype(self, data_type, operation_id=None):
"""
Launch adapter to store a create a persistent DataType.
"""
operation_id = operation_id or self.operation.id
data_type.type = data_type.__class__.__name__
data_type.module = data_type.__class__.__module__
data_type.subject = self.USER_FULL_NAME
data_type.state = self.DATATYPE_STATE
data_type.set_operation_id(operation_id)
adapter_instance = StoreAdapter([data_type])
operation = dao.get_operation_by_id(operation_id)
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return data_type
def create_simple_datatype(self, subject=USER_FULL_NAME, state=DATATYPE_STATE):
"""
This method creates a simple data type
"""
datatype_inst = Datatype1()
self._fill_datatype(datatype_inst, subject, state)
# Store data type
return self._store_datatype(datatype_inst)
def create_datatype_with_storage(self, subject=USER_FULL_NAME, state=DATATYPE_STATE,
data=DATATYPE_DATA, operation_id=None):
"""
This method creates and stores a data type which imply storage on the file system.
"""
datatype_inst = Datatype2()
self._fill_datatype(datatype_inst, subject, state, operation_id)
datatype_inst.string_data = data
return self._store_datatype(datatype_inst, operation_id)
def _fill_datatype(self, datatype, subject, state, operation_id=None):
"""
This method sets some common attributes on dataType
"""
operation_id = operation_id or self.operation.id
datatype.subject = subject
datatype.state = state
# Set_operation_id also sets storage_path attribute
datatype.set_operation_id(operation_id)
def __create_operation(self):
"""
Create a operation entity. Return the operation, algo_id and the storage path.
"""
meta = {DataTypeMetaData.KEY_SUBJECT: "John Doe", DataTypeMetaData.KEY_STATE: "RAW_DATA"}
algorithm, algo_group = FlowService().get_algorithm_by_module_and_class(SIMULATOR_MODULE, SIMULATOR_CLASS)
operation = model.Operation(self.user.id, self.project.id, algo_group.id, json.dumps(''), meta=json.dumps(meta),
status=model.STATUS_STARTED, method_name=ABCAdapter.LAUNCH_METHOD)
operation = dao.store_entity(operation)
storage_path = FilesHelper().get_project_folder(self.project, str(operation.id))
return operation, algorithm.id, storage_path
def create_connectivity(self):
"""
Create a connectivity that will be used in "non-dummy" burst launches (with the actual simulator).
"""
operation, algo_id, storage_path = self.__create_operation()
connectivity = Connectivity(storage_path=storage_path)
connectivity.weights = numpy.ones((74, 74))
connectivity.centres = numpy.ones((74, 3))
adapter_instance = StoreAdapter([connectivity])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return algo_id, connectivity
def create_timeseries(self, connectivity, ts_type=None, sensors=None):
"""
Create a stored TimeSeries entity.
"""
operation, _, storage_path = self.__create_operation()
if ts_type == "EEG":
time_series = TimeSeriesEEG(storage_path=storage_path, sensors=sensors)
else:
rm = dao.get_generic_entity(RegionMapping, connectivity.gid, '_connectivity')
if len(rm) < 1:
rm = None
else:
rm = rm[0]
time_series = TimeSeriesRegion(storage_path=storage_path, connectivity=connectivity, region_mapping=rm)
data = numpy.random.random((10, 10, 10, 10))
time_series.write_data_slice(data)
time_series.write_time_slice(numpy.arange(10))
adapter_instance = StoreAdapter([time_series])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
time_series = dao.get_datatype_by_gid(time_series.gid)
return time_series
def create_covariance(self, time_series):
"""
:returns: a stored DataType Covariance.
"""
operation, _, storage_path = self.__create_operation()
covariance = Covariance(storage_path=storage_path, source=time_series)
covariance.write_data_slice(numpy.random.random((10, 10, 10)))
adapter_instance = StoreAdapter([covariance])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return covariance
def create_crosscoherence(self, time_series):
"""
:returns: a stored entity of type CoherenceSpectrum
"""
operation, _, storage_path = self.__create_operation()
partial_coh = CoherenceSpectrum(array_data=numpy.random.random((10, 10, 10, 10)), use_storage=False)
coherence = CoherenceSpectrum(source=time_series, storage_path=storage_path, frequency=0.1, nfft=256)
coherence.write_data_slice(partial_coh)
coherence.close_file()
adapter_instance = StoreAdapter([coherence])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return coherence
def create_crosscorrelation(self, time_series):
"""
:returns: `CrossCorrelation` stored entity.
"""
operation, _, storage_path = self.__create_operation()
partial_corr = CrossCorrelation(array_data=numpy.random.random((10, 10, 10, 10, 10)), use_storage=False)
crossc = CrossCorrelation(source=time_series, storage_path=storage_path, time=range(10))
crossc.write_data_slice(partial_corr)
crossc.close_file()
adapter_instance = StoreAdapter([crossc])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return crossc
def create_surface(self):
"""
Create a dummy surface entity.
:returns: (Algorithm Identifier, stored Surface entity)
"""
operation, algo_id, storage_path = self.__create_operation()
surface = CorticalSurface(storage_path=storage_path)
surface.vertices = numpy.array([[-10, 0, 0],
[0, 0, -10],
[10, 0, 0],
[0, 10, 0]], dtype=float)
surface.triangles = numpy.array([[0, 1, 2],
[0, 1, 3],
[1, 2, 3],
[0, 2, 3]], dtype=int)
surface.number_of_triangles = 4
surface.number_of_vertices = 4
surface.triangle_normals = numpy.ones((4, 3))
surface.vertex_normals = numpy.ones((4, 3))
surface.zero_based_triangles = True
surface.validate()
adapter_instance = StoreAdapter([surface])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return algo_id, surface
def create_connectivity_measure(self, connectivity):
"""
:returns: persisted entity ConnectivityMeasure
"""
operation, _, storage_path = self.__create_operation()
conn_measure = ConnectivityMeasure(storage_path=storage_path)
conn_measure.connectivity = connectivity
adapter_instance = StoreAdapter([conn_measure])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return conn_measure
def create_datatype_measure(self, analyzed_entity, operation=None, storage_path=None):
"""
:return: persisted DatatypeMeasure
"""
if operation is None:
operation, _, storage_path = self.__create_operation()
measure = DatatypeMeasure(storage_path=storage_path, metrics=self.DATATYPE_MEASURE_METRIC)
measure.analyzed_datatype = analyzed_entity
adapter_instance = StoreAdapter([measure])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return measure
def create_ICA(self, timeseries):
"""
:returns: persisted entity IndependentComponents
"""
operation, _, storage_path = self.__create_operation()
partial_ts = TimeSeries(use_storage=False)
partial_ts.data = numpy.random.random((10, 10, 10, 10))
partial_ica = IndependentComponents(source=partial_ts,
component_time_series=numpy.random.random((10, 10, 10, 10)),
prewhitening_matrix=numpy.random.random((10, 10, 10, 10)),
unmixing_matrix=numpy.random.random((10, 10, 10, 10)),
n_components=10, use_storage=False)
ica = IndependentComponents(source=timeseries, n_components=10, storage_path=storage_path)
ica.write_data_slice(partial_ica)
adapter_instance = StoreAdapter([ica])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return ica
def create_datatype_group(self, subject=USER_FULL_NAME, state=DATATYPE_STATE, ):
"""
This method creates, stores and returns a DataTypeGroup entity.
"""
group = model.OperationGroup(self.project.id, ranges=[json.dumps(self.RANGE_1), json.dumps(self.RANGE_2)])
group = dao.store_entity(group)
group_ms = model.OperationGroup(self.project.id, ranges=[json.dumps(self.RANGE_1), json.dumps(self.RANGE_2)])
group_ms = dao.store_entity(group_ms)
datatype_group = model.DataTypeGroup(group, subject=subject, state=state, operation_id=self.operation.id)
# Set storage path, before setting data
datatype_group.storage_path = self.files_helper.get_project_folder(self.project, str(self.operation.id))
datatype_group = dao.store_entity(datatype_group)
dt_group_ms = model.DataTypeGroup(group_ms, subject=subject, state=state, operation_id=self.operation.id)
# Set storage path, before setting data
dt_group_ms.storage_path = self.files_helper.get_project_folder(self.project, str(self.operation.id))
dao.store_entity(dt_group_ms)
# Now create some data types and add them to group
for range_val1 in self.RANGE_1[1]:
for range_val2 in self.RANGE_2[1]:
operation = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters',
meta=json.dumps(self.meta), status=model.STATUS_FINISHED,
method_name=ABCAdapter.LAUNCH_METHOD,
range_values=json.dumps({self.RANGE_1[0]: range_val1,
self.RANGE_2[0]: range_val2}))
operation.fk_operation_group = group.id
operation = dao.store_entity(operation)
datatype = self.create_datatype_with_storage(operation_id=operation.id)
datatype.number1 = range_val1
datatype.number2 = range_val2
datatype.fk_datatype_group = datatype_group.id
datatype.set_operation_id(operation.id)
dao.store_entity(datatype)
op_ms = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters',
meta=json.dumps(self.meta), status=model.STATUS_FINISHED,
method_name=ABCAdapter.LAUNCH_METHOD,
range_values=json.dumps({self.RANGE_1[0]: range_val1,
self.RANGE_2[0]: range_val2}))
op_ms.fk_operation_group = group_ms.id
op_ms = dao.store_entity(op_ms)
self.create_datatype_measure(datatype, op_ms,
FilesHelper().get_project_folder(self.project, str(op_ms.id)))
return datatype_group
class OperationFacade:
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.operation_service = OperationService()
self.project_service = ProjectService()
self.user_service = UserService()
self.files_helper = FilesHelper()
@staticmethod
def get_operation_status(operation_gid):
operation = ProjectService.load_operation_by_gid(operation_gid)
if operation is None:
get_logger().warning(
"Invalid operation GID: {}".format(operation_gid))
raise InvalidIdentifierException()
return operation.status
@staticmethod
def get_operations_results(operation_gid):
operation = ProjectService.load_operation_lazy_by_gid(operation_gid)
if operation is None:
get_logger().warning(
"Invalid operation GID: {}".format(operation_gid))
raise InvalidIdentifierException()
data_types = ProjectService.get_results_for_operation(operation.id)
if data_types is None:
return []
return [DataTypeDto(datatype) for datatype in data_types]
def launch_operation(self, current_user_id, model_file, project_gid,
algorithm_module, algorithm_classname, fetch_file):
temp_folder = FilesHelper.create_temp_folder()
model_h5_path = FilesHelper.save_temporary_file(
model_file, temp_folder)
try:
project = self.project_service.find_project_lazy_by_gid(
project_gid)
except ProjectServiceException:
raise InvalidIdentifierException()
algorithm = AlgorithmService.get_algorithm_by_module_and_class(
algorithm_module, algorithm_classname)
if algorithm is None:
raise InvalidIdentifierException(
'No algorithm found for: %s.%s' %
(algorithm_module, algorithm_classname))
try:
adapter_instance = ABCAdapter.build_adapter(algorithm)
view_model = adapter_instance.get_view_model_class()()
view_model_h5 = ViewModelH5(model_h5_path, view_model)
view_model_gid = view_model_h5.gid.load()
operation = self.operation_service.prepare_operation(
current_user_id, project.id, algorithm, view_model_gid.hex)
storage_path = self.files_helper.get_project_folder(
project, str(operation.id))
if isinstance(adapter_instance, ABCUploader):
for key, value in adapter_instance.get_form_class(
).get_upload_information().items():
data_file = fetch_file(request_file_key=key,
file_extension=value)
data_file_path = FilesHelper.save_temporary_file(
data_file, temp_folder)
file_name = os.path.basename(data_file_path)
upload_field = getattr(view_model_h5, key)
upload_field.store(os.path.join(storage_path, file_name))
shutil.move(data_file_path, storage_path)
shutil.move(model_h5_path, storage_path)
os.rmdir(temp_folder)
view_model_h5.close()
OperationService().launch_operation(operation.id, True)
return operation.gid
except Exception as excep:
self.logger.error(excep, exc_info=True)
raise ServiceException(str(excep))
class ImportService(object):
"""
Service for importing TVB entities into system.
It supports TVB exported H5 files as input, but it should also handle H5 files
generated outside of TVB, as long as they respect the same structure.
"""
def __init__(self):
self.logger = get_logger(__name__)
self.user_id = None
self.files_helper = FilesHelper()
self.created_projects = []
def _download_and_unpack_project_zip(self, uploaded, uq_file_name,
temp_folder):
if isinstance(uploaded, FieldStorage) or isinstance(uploaded, Part):
if not uploaded.file:
raise ImportException(
"Please select the archive which contains the project structure."
)
with open(uq_file_name, 'wb') as file_obj:
self.files_helper.copy_file(uploaded.file, file_obj)
else:
shutil.copy2(uploaded, uq_file_name)
try:
self.files_helper.unpack_zip(uq_file_name, temp_folder)
except FileStructureException as excep:
self.logger.exception(excep)
raise ImportException(
"Bad ZIP archive provided. A TVB exported project is expected!"
)
@staticmethod
def _compute_unpack_path():
"""
:return: the name of the folder where to expand uploaded zip
"""
now = datetime.now()
date_str = "%d-%d-%d_%d-%d-%d_%d" % (now.year, now.month, now.day,
now.hour, now.minute, now.second,
now.microsecond)
uq_name = "%s-ImportProject" % date_str
return os.path.join(TvbProfile.current.TVB_TEMP_FOLDER, uq_name)
@transactional
def import_project_structure(self, uploaded, user_id):
"""
Execute import operations:
1. check if ZIP or folder
2. find all project nodes
3. for each project node:
- create project
- create all operations and groups
- import all images
- create all dataTypes
"""
self.user_id = user_id
self.created_projects = []
# Now compute the name of the folder where to explode uploaded ZIP file
temp_folder = self._compute_unpack_path()
uq_file_name = temp_folder + ".zip"
try:
self._download_and_unpack_project_zip(uploaded, uq_file_name,
temp_folder)
self._import_projects_from_folder(temp_folder)
except Exception as excep:
self.logger.exception(
"Error encountered during import. Deleting projects created during this operation."
)
# Remove project folders created so far.
# Note that using the project service to remove the projects will not work,
# because we do not have support for nested transaction.
# Removing from DB is not necessary because in transactional env a simple exception throw
# will erase everything to be inserted.
for project in self.created_projects:
project_path = os.path.join(TvbProfile.current.TVB_STORAGE,
FilesHelper.PROJECTS_FOLDER,
project.name)
shutil.rmtree(project_path)
raise ImportException(str(excep))
finally:
# Now delete uploaded file
if os.path.exists(uq_file_name):
os.remove(uq_file_name)
# Now delete temporary folder where uploaded ZIP was exploded.
if os.path.exists(temp_folder):
shutil.rmtree(temp_folder)
def _import_projects_from_folder(self, temp_folder):
"""
Process each project from the uploaded pack, to extract names.
"""
project_roots = []
for root, _, files in os.walk(temp_folder):
if FilesHelper.TVB_PROJECT_FILE in files:
project_roots.append(root)
for temp_project_path in project_roots:
update_manager = ProjectUpdateManager(temp_project_path)
update_manager.run_all_updates()
project = self.__populate_project(temp_project_path)
# Populate the internal list of create projects so far, for cleaning up folders, in case of failure
self.created_projects.append(project)
# Ensure project final folder exists on disk
project_path = self.files_helper.get_project_folder(project)
shutil.move(
os.path.join(temp_project_path, FilesHelper.TVB_PROJECT_FILE),
project_path)
# Now import project operations with their results
self.import_project_operations(project, temp_project_path)
# Import images and move them from temp into target
self._store_imported_images(project, temp_project_path,
project.name)
def _load_datatypes_from_operation_folder(self, src_op_path,
operation_entity,
datatype_group):
"""
Loads datatypes from operation folder
:returns: Datatype entities as dict {original_path: Dt instance}
"""
all_datatypes = {}
for file_name in os.listdir(src_op_path):
if file_name.endswith(FilesHelper.TVB_STORAGE_FILE_EXTENSION):
h5_file = os.path.join(src_op_path, file_name)
try:
file_update_manager = FilesUpdateManager()
file_update_manager.upgrade_file(h5_file)
datatype = self.load_datatype_from_file(
h5_file, operation_entity.id, datatype_group,
operation_entity.fk_launched_in)
all_datatypes[h5_file] = datatype
except IncompatibleFileManagerException:
os.remove(h5_file)
self.logger.warning(
"Incompatible H5 file will be ignored: %s" % h5_file)
self.logger.exception("Incompatibility details ...")
return all_datatypes
def _store_imported_datatypes_in_db(self, project, all_datatypes):
# type: (Project, dict) -> int
sorted_dts = sorted(
all_datatypes.items(),
key=lambda dt_item: dt_item[1].create_date or datetime.now())
count = 0
for dt_path, datatype in sorted_dts:
datatype_already_in_tvb = dao.get_datatype_by_gid(datatype.gid)
if not datatype_already_in_tvb:
self.store_datatype(datatype, dt_path)
count += 1
else:
AlgorithmService.create_link([datatype_already_in_tvb.id],
project.id)
file_path = h5.h5_file_for_index(datatype).path
h5_class = H5File.h5_class_from_file(file_path)
reference_list = h5_class(file_path).gather_references()
for _, reference_gid in reference_list:
if not reference_gid:
continue
ref_index = dao.get_datatype_by_gid(reference_gid.hex)
if ref_index is None:
os.remove(file_path)
dao.remove_entity(datatype.__class__, datatype.id)
raise MissingReferenceException(
'Imported file depends on datatypes that do not exist. Please upload '
'those first!')
return count
def _store_imported_images(self, project, temp_project_path, project_name):
"""
Import all images from project
"""
images_root = os.path.join(temp_project_path,
FilesHelper.IMAGES_FOLDER)
target_images_path = self.files_helper.get_images_folder(project_name)
for root, _, files in os.walk(images_root):
for metadata_file in files:
if metadata_file.endswith(FilesHelper.TVB_FILE_EXTENSION):
self._import_image(root, metadata_file, project.id,
target_images_path)
def _retrieve_operations_in_order(self, project, import_path):
# type: (Project, str) -> list[Operation2ImportData]
retrieved_operations = []
for root, _, files in os.walk(import_path):
if OPERATION_XML in files:
# Previous Operation format for uploading previous versions of projects
operation_file_path = os.path.join(root, OPERATION_XML)
operation, operation_xml_parameters = self.__build_operation_from_file(
project, operation_file_path)
operation.import_file = operation_file_path
self.logger.debug("Found operation in old XML format: " +
str(operation))
retrieved_operations.append(
Operation2ImportData(
operation,
root,
info_from_xml=operation_xml_parameters))
else:
# We strive for the new format with ViewModelH5
main_view_model = None
dt_paths = []
all_view_model_files = []
for file in files:
if file.endswith(FilesHelper.TVB_STORAGE_FILE_EXTENSION):
h5_file = os.path.join(root, file)
try:
h5_class = H5File.h5_class_from_file(h5_file)
if h5_class is ViewModelH5:
all_view_model_files.append(h5_file)
if not main_view_model:
view_model = h5.load_view_model_from_file(
h5_file)
if type(view_model
) in VIEW_MODEL2ADAPTER.keys():
main_view_model = view_model
else:
file_update_manager = FilesUpdateManager()
file_update_manager.upgrade_file(h5_file)
dt_paths.append(h5_file)
except Exception:
self.logger.warning(
"Unreadable H5 file will be ignored: %s" %
h5_file)
if main_view_model is not None:
alg = VIEW_MODEL2ADAPTER[type(main_view_model)]
operation = Operation(main_view_model.gid.hex,
project.fk_admin,
project.id,
alg.id,
status=STATUS_FINISHED,
user_group=main_view_model.
generic_attributes.operation_tag,
start_date=datetime.now(),
completion_date=datetime.now())
operation.create_date = main_view_model.create_date
self.logger.debug(
"Found main ViewModel to create operation for it: " +
str(operation))
retrieved_operations.append(
Operation2ImportData(operation, root, main_view_model,
dt_paths, all_view_model_files))
elif len(dt_paths) > 0:
alg = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE,
TVB_IMPORTER_CLASS)
default_adapter = ABCAdapter.build_adapter(alg)
view_model = default_adapter.get_view_model_class()()
view_model.data_file = dt_paths[0]
vm_path = h5.store_view_model(view_model, root)
all_view_model_files.append(vm_path)
operation = Operation(view_model.gid.hex,
project.fk_admin,
project.id,
alg.id,
status=STATUS_FINISHED,
start_date=datetime.now(),
completion_date=datetime.now())
self.logger.debug(
"Found no ViewModel in folder, so we default to " +
str(operation))
retrieved_operations.append(
Operation2ImportData(operation, root, view_model,
dt_paths, all_view_model_files,
True))
return sorted(retrieved_operations,
key=lambda op_data: op_data.order_field)
def import_project_operations(self, project, import_path):
"""
This method scans provided folder and identify all operations that needs to be imported
"""
imported_operations = []
ordered_operations = self._retrieve_operations_in_order(
project, import_path)
success_no = 0
for operation_data in ordered_operations:
if operation_data.is_old_form:
operation_entity, datatype_group = self.__import_operation(
operation_data.operation)
new_op_folder = self.files_helper.get_project_folder(
project, str(operation_entity.id))
try:
operation_datatypes = self._load_datatypes_from_operation_folder(
operation_data.operation_folder, operation_entity,
datatype_group)
# Create and store view_model from operation
view_model = self._get_new_form_view_model(
operation_entity, operation_data.info_from_xml)
h5.store_view_model(view_model, new_op_folder)
operation_entity.view_model_gid = view_model.gid.hex
dao.store_entity(operation_entity)
self._store_imported_datatypes_in_db(
project, operation_datatypes)
imported_operations.append(operation_entity)
success_no = success_no + 1
except MissingReferenceException:
operation_entity.status = STATUS_ERROR
dao.store_entity(operation_entity)
elif operation_data.main_view_model is not None:
operation_entity = dao.store_entity(operation_data.operation)
dt_group = None # TODO
# Store the DataTypes in db
dts = {}
for dt_path in operation_data.dt_paths:
dt = self.load_datatype_from_file(dt_path,
operation_entity.id,
dt_group, project.id)
if isinstance(dt, BurstConfiguration):
dao.store_entity(dt)
else:
dts[dt_path] = dt
try:
stored_dts_count = self._store_imported_datatypes_in_db(
project, dts)
if stored_dts_count > 0 or not operation_data.is_self_generated:
imported_operations.append(operation_entity)
new_op_folder = self.files_helper.get_project_folder(
project, str(operation_entity.id))
for h5_file in operation_data.all_view_model_files:
shutil.move(h5_file, new_op_folder)
else:
# In case all Dts under the current operation were Links and the ViewModel is dummy,
# don't keep the Operation empty in DB
dao.remove_entity(Operation, operation_entity.id)
except MissingReferenceException:
operation_entity.status = STATUS_ERROR
dao.store_entity(operation_entity)
else:
self.logger.warning(
"Folder %s will be ignored, as we could not find a serialized "
"operation or DTs inside!" %
operation_data.operation_folder)
self.logger.warning(
"Project has been only partially imported because of some missing dependent datatypes. "
+ "%d files were successfully imported from a total of %d!" %
(success_no, len(ordered_operations)))
return imported_operations
@staticmethod
def _get_new_form_view_model(operation, xml_parameters):
# type (Operation) -> ViewModel
ad = ABCAdapter.build_adapter(operation.algorithm)
view_model = ad.get_view_model_class()()
if xml_parameters:
params = json.loads(xml_parameters)
declarative_attrs = type(view_model).declarative_attrs
for param in params:
new_param_name = param
if param[0] == "_":
new_param_name = param[1:]
new_param_name = new_param_name.lower()
if new_param_name in declarative_attrs:
setattr(view_model, new_param_name, params[param])
return view_model
def _import_image(self, src_folder, metadata_file, project_id,
target_images_path):
"""
Create and store a image entity.
"""
figure_dict = XMLReader(os.path.join(src_folder,
metadata_file)).read_metadata()
actual_figure = os.path.join(
src_folder,
os.path.split(figure_dict['file_path'])[1])
if not os.path.exists(actual_figure):
self.logger.warning("Expected to find image path %s .Skipping" %
actual_figure)
return
figure_dict['fk_user_id'] = self.user_id
figure_dict['fk_project_id'] = project_id
figure_entity = manager_of_class(ResultFigure).new_instance()
figure_entity = figure_entity.from_dict(figure_dict)
stored_entity = dao.store_entity(figure_entity)
# Update image meta-data with the new details after import
figure = dao.load_figure(stored_entity.id)
shutil.move(actual_figure, target_images_path)
self.logger.debug("Store imported figure")
self.files_helper.write_image_metadata(figure)
def load_datatype_from_file(self,
current_file,
op_id,
datatype_group=None,
current_project_id=None):
# type: (str, int, DataTypeGroup, int) -> HasTraitsIndex
"""
Creates an instance of datatype from storage / H5 file
:returns: DatatypeIndex
"""
self.logger.debug("Loading DataType from file: %s" % current_file)
h5_class = H5File.h5_class_from_file(current_file)
if h5_class is BurstConfigurationH5:
if current_project_id is None:
op_entity = dao.get_operationgroup_by_id(op_id)
current_project_id = op_entity.fk_launched_in
h5_file = BurstConfigurationH5(current_file)
burst = BurstConfiguration(current_project_id)
burst.fk_simulation = op_id
# burst.fk_operation_group = TODO
# burst.fk_metric_operation_group = TODO
h5_file.load_into(burst)
result = burst
else:
datatype, generic_attributes = h5.load_with_links(current_file)
index_class = h5.REGISTRY.get_index_for_datatype(
datatype.__class__)
datatype_index = index_class()
datatype_index.fill_from_has_traits(datatype)
datatype_index.fill_from_generic_attributes(generic_attributes)
# Add all the required attributes
if datatype_group is not None:
datatype_index.fk_datatype_group = datatype_group.id
datatype_index.fk_from_operation = op_id
associated_file = h5.path_for_stored_index(datatype_index)
if os.path.exists(associated_file):
datatype_index.disk_size = FilesHelper.compute_size_on_disk(
associated_file)
result = datatype_index
return result
def store_datatype(self, datatype, current_file=None):
"""This method stores data type into DB"""
try:
self.logger.debug("Store datatype: %s with Gid: %s" %
(datatype.__class__.__name__, datatype.gid))
# Now move storage file into correct folder if necessary
if current_file is not None:
final_path = h5.path_for_stored_index(datatype)
if final_path != current_file:
shutil.move(current_file, final_path)
return dao.store_entity(datatype)
except MissingDataSetException as e:
self.logger.exception(e)
error_msg = "Datatype %s has missing data and could not be imported properly." % (
datatype, )
raise ImportException(error_msg)
except IntegrityError as excep:
self.logger.exception(excep)
error_msg = "Could not import data with gid: %s. There is already a one with " \
"the same name or gid." % datatype.gid
raise ImportException(error_msg)
def __populate_project(self, project_path):
"""
Create and store a Project entity.
"""
self.logger.debug("Creating project from path: %s" % project_path)
project_dict = self.files_helper.read_project_metadata(project_path)
project_entity = manager_of_class(Project).new_instance()
project_entity = project_entity.from_dict(project_dict, self.user_id)
try:
self.logger.debug("Storing imported project")
return dao.store_entity(project_entity)
except IntegrityError as excep:
self.logger.exception(excep)
error_msg = (
"Could not import project: %s with gid: %s. There is already a "
"project with the same name or gid.") % (project_entity.name,
project_entity.gid)
raise ImportException(error_msg)
def __build_operation_from_file(self, project, operation_file):
"""
Create Operation entity from metadata file.
"""
operation_dict = XMLReader(operation_file).read_metadata()
operation_entity = manager_of_class(Operation).new_instance()
return operation_entity.from_dict(operation_dict, dao, self.user_id,
project.gid)
@staticmethod
def __import_operation(operation_entity):
"""
Store a Operation entity.
"""
operation_entity = dao.store_entity(operation_entity)
operation_group_id = operation_entity.fk_operation_group
datatype_group = None
if operation_group_id is not None:
try:
datatype_group = dao.get_datatypegroup_by_op_group_id(
operation_group_id)
except SQLAlchemyError:
# If no dataType group present for current op. group, create it.
operation_group = dao.get_operationgroup_by_id(
operation_group_id)
datatype_group = DataTypeGroup(
operation_group, operation_id=operation_entity.id)
datatype_group.state = UploadAlgorithmCategoryConfig.defaultdatastate
datatype_group = dao.store_entity(datatype_group)
return operation_entity, datatype_group
def import_simulator_configuration_zip(self, zip_file):
# Now compute the name of the folder where to explode uploaded ZIP file
temp_folder = self._compute_unpack_path()
uq_file_name = temp_folder + ".zip"
if isinstance(zip_file, FieldStorage) or isinstance(zip_file, Part):
if not zip_file.file:
raise ServicesBaseException(
"Could not process the given ZIP file...")
with open(uq_file_name, 'wb') as file_obj:
self.files_helper.copy_file(zip_file.file, file_obj)
else:
shutil.copy2(zip_file, uq_file_name)
try:
self.files_helper.unpack_zip(uq_file_name, temp_folder)
return temp_folder
except FileStructureException as excep:
raise ServicesBaseException(
"Could not process the given ZIP file..." + str(excep))
# # Sort all h5 files based on their creation date stored in the files themselves
# sorted_h5_files = sorted(h5_files, key=lambda h5_path: _get_create_date_for_sorting(h5_path) or datetime.now())
# return sorted_h5_files
#
#
# def _get_create_date_for_sorting(h5_file):
# storage_manager = HDF5StorageManager(os.path.dirname(h5_file), os.path.basename(h5_file))
# root_metadata = storage_manager.get_metadata()
# create_date_str = str(root_metadata['Create_date'], 'utf-8')
# create_date = datetime.strptime(create_date_str.replace('datetime:', ''), '%Y-%m-%d %H:%M:%S.%f')
# return create_date
class ProjectService:
"""
Services layer for Project entities.
"""
def __init__(self):
self.logger = get_logger(__name__)
self.structure_helper = FilesHelper()
def store_project(self, current_user, is_create, selected_id, **data):
"""
We want to create/update a project entity.
"""
# Validate Unique Name
new_name = data["name"]
if len(new_name) < 1:
raise ProjectServiceException("Invalid project name!")
projects_no = dao.count_projects_for_name(new_name, selected_id)
if projects_no > 0:
err = {'name': 'Please choose another name, this one is used!'}
raise formencode.Invalid("Duplicate Name Error", {}, None, error_dict=err)
started_operations = dao.get_operation_numbers(selected_id)[1]
if started_operations > 0:
raise ProjectServiceException("A project can not be renamed while operations are still running!")
if is_create:
current_proj = Project(new_name, current_user.id, data["description"])
self.structure_helper.get_project_folder(current_proj)
else:
try:
current_proj = dao.get_project_by_id(selected_id)
except Exception as excep:
self.logger.exception("An error has occurred!")
raise ProjectServiceException(str(excep))
if current_proj.name != new_name:
self.structure_helper.rename_project_structure(current_proj.name, new_name)
current_proj.name = new_name
current_proj.description = data["description"]
# Commit to make sure we have a valid ID
current_proj.refresh_update_date()
self.structure_helper.write_project_metadata(current_proj)
current_proj = dao.store_entity(current_proj)
# Retrieve, to initialize lazy attributes
current_proj = dao.get_project_by_id(current_proj.id)
# Update share settings on current Project entity
visited_pages = []
prj_admin = current_proj.administrator.username
if 'visited_pages' in data and data['visited_pages']:
visited_pages = data['visited_pages'].split(',')
for page in visited_pages:
members = UserService.retrieve_users_except(prj_admin, int(page), MEMBERS_PAGE_SIZE)[0]
members = [m.id for m in members]
dao.delete_members_for_project(current_proj.id, members)
selected_user_ids = data["users"]
if is_create and current_user.id not in selected_user_ids:
# Make the project admin also member of the current project
selected_user_ids.append(current_user.id)
dao.add_members_to_project(current_proj.id, selected_user_ids)
# Finish operation
self.logger.debug("Edit/Save OK for project:" + str(current_proj.id) + ' by user:'******'-'
result["count"] = one_op[2]
result["gid"] = one_op[13]
if one_op[3] is not None and one_op[3]:
try:
operation_group = dao.get_generic_entity(OperationGroup, one_op[3])[0]
result["group"] = operation_group.name
result["group"] = result["group"].replace("_", " ")
result["operation_group_id"] = operation_group.id
datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3])
result["datatype_group_gid"] = datatype_group.gid
result["gid"] = operation_group.gid
## Filter only viewers for current DataTypeGroup entity:
result["view_groups"] = FlowService().get_visualizers_for_group(datatype_group.gid)
except Exception:
self.logger.exception("We will ignore group on entity:" + str(one_op))
result["datatype_group_gid"] = None
else:
result['group'] = None
result['datatype_group_gid'] = None
result["algorithm"] = dao.get_algorithm_by_id(one_op[4])
result["user"] = dao.get_user_by_id(one_op[5])
if type(one_op[6]) is str:
result["create"] = string2date(str(one_op[6]))
else:
result["create"] = one_op[6]
if type(one_op[7]) is str:
result["start"] = string2date(str(one_op[7]))
else:
result["start"] = one_op[7]
if type(one_op[8]) is str:
result["complete"] = string2date(str(one_op[8]))
else:
result["complete"] = one_op[8]
if result["complete"] is not None and result["start"] is not None:
result["duration"] = format_timedelta(result["complete"] - result["start"])
result["status"] = one_op[9]
result["additional"] = one_op[10]
result["visible"] = True if one_op[11] > 0 else False
result['operation_tag'] = one_op[12]
result['figures'] = None
if not result['group']:
datatype_results = dao.get_results_for_operation(result['id'])
result['results'] = []
for dt in datatype_results:
dt_loaded = ABCAdapter.load_entity_by_gid(dt.gid)
if dt_loaded:
result['results'].append(dt_loaded)
else:
self.logger.warning("Could not retrieve datatype %s" % str(dt))
operation_figures = dao.get_figures_for_operation(result['id'])
# Compute the full path to the figure / image on disk
for figure in operation_figures:
figures_folder = self.structure_helper.get_images_folder(figure.project.name)
figure_full_path = os.path.join(figures_folder, figure.file_path)
# Compute the path available from browser
figure.figure_path = utils.path2url_part(figure_full_path)
result['figures'] = operation_figures
else:
result['results'] = None
operations.append(result)
except Exception:
## We got an exception when processing one Operation Row. We will continue with the rest of the rows.
self.logger.exception("Could not prepare operation for display:" + str(one_op))
return selected_project, total_ops_nr, operations, pages_no
def retrieve_projects_for_user(self, user_id, current_page=1):
"""
Return a list with all Projects visible for current user.
"""
start_idx = PROJECTS_PAGE_SIZE * (current_page - 1)
total = dao.get_projects_for_user(user_id, is_count=True)
available_projects = dao.get_projects_for_user(user_id, start_idx, PROJECTS_PAGE_SIZE)
pages_no = total // PROJECTS_PAGE_SIZE + (1 if total % PROJECTS_PAGE_SIZE else 0)
for prj in available_projects:
fns, sta, err, canceled, pending = dao.get_operation_numbers(prj.id)
prj.operations_finished = fns
prj.operations_started = sta
prj.operations_error = err
prj.operations_canceled = canceled
prj.operations_pending = pending
prj.disk_size = dao.get_project_disk_size(prj.id)
prj.disk_size_human = format_bytes_human(prj.disk_size)
self.logger.debug("Displaying " + str(len(available_projects)) + " projects in UI for user " + str(user_id))
return available_projects, pages_no
@staticmethod
def retrieve_all_user_projects(user_id, page_start=0, page_size= PROJECTS_PAGE_SIZE):
"""
Return a list with all projects visible for current user, without pagination.
"""
return dao.get_projects_for_user(user_id, page_start=page_start, page_size=page_size)
@staticmethod
def get_linkable_projects_for_user(user_id, data_id):
"""
Find projects with are visible for current user, and in which current datatype hasn't been linked yet.
"""
return dao.get_linkable_projects_for_user(user_id, data_id)
@transactional
def remove_project(self, project_id):
"""
Remove Project from DB and File Storage.
"""
try:
project2delete = dao.get_project_by_id(project_id)
self.logger.debug("Deleting project: id=" + str(project_id) + ' name=' + project2delete.name)
project_bursts = dao.get_bursts_for_project(project_id)
for burst in project_bursts:
dao.remove_entity(burst.__class__, burst.id)
project_datatypes = dao.get_datatypes_in_project(project_id)
for one_data in project_datatypes:
self.remove_datatype(project_id, one_data.gid, True)
links = dao.get_links_for_project(project_id)
for one_link in links:
dao.remove_entity(Links, one_link.id)
self.structure_helper.remove_project_structure(project2delete.name)
dao.delete_project(project_id)
self.logger.debug("Deleted project: id=" + str(project_id) + ' name=' + project2delete.name)
except RemoveDataTypeException as excep:
self.logger.exception("Could not execute operation Node Remove!")
raise ProjectServiceException(str(excep))
except FileStructureException as excep:
self.logger.exception("Could not delete because of rights!")
raise ProjectServiceException(str(excep))
except Exception as excep:
self.logger.exception(str(excep))
raise ProjectServiceException(str(excep))
# ----------------- Methods for populating Data-Structure Page ---------------
@staticmethod
def get_datatype_in_group(group):
"""
Return all dataTypes that are the result of the same DTgroup.
"""
return dao.get_datatype_in_group(datatype_group_id=group)
@staticmethod
def get_datatypes_from_datatype_group(datatype_group_id):
"""
Retrieve all dataType which are part from the given dataType group.
"""
return dao.get_datatypes_from_datatype_group(datatype_group_id)
@staticmethod
def load_operation_by_gid(operation_gid):
""" Retrieve loaded Operation from DB"""
return dao.get_operation_by_gid(operation_gid)
@staticmethod
def load_operation_lazy_by_gid(operation_gid):
""" Retrieve lazy Operation from DB"""
return dao.get_operation_lazy_by_gid(operation_gid)
@staticmethod
def get_operation_group_by_id(operation_group_id):
""" Loads OperationGroup from DB"""
return dao.get_operationgroup_by_id(operation_group_id)
@staticmethod
def get_operation_group_by_gid(operation_group_gid):
""" Loads OperationGroup from DB"""
return dao.get_operationgroup_by_gid(operation_group_gid)
@staticmethod
def get_operations_in_group(operation_group):
""" Return all the operations from an operation group. """
return dao.get_operations_in_group(operation_group.id)
@staticmethod
def is_upload_operation(operation_gid):
""" Returns True only if the operation with the given GID is an upload operation. """
return dao.is_upload_operation(operation_gid)
@staticmethod
def get_all_operations_for_uploaders(project_id):
""" Returns all finished upload operations. """
return dao.get_all_operations_for_uploaders(project_id)
def set_operation_and_group_visibility(self, entity_gid, is_visible, is_operation_group=False):
"""
Sets the operation visibility.
If 'is_operation_group' is True than this method will change the visibility for all
the operation from the OperationGroup with the GID field equal to 'entity_gid'.
"""
def set_visibility(op):
# workaround:
# 'reload' the operation so that it has the project property set.
# get_operations_in_group does not eager load it and now we're out of a sqlalchemy session
# write_operation_metadata requires that property
op = dao.get_operation_by_id(op.id)
# end hack
op.visible = is_visible
self.structure_helper.write_operation_metadata(op)
dao.store_entity(op)
def set_group_descendants_visibility(operation_group_id):
ops_in_group = dao.get_operations_in_group(operation_group_id)
for group_op in ops_in_group:
set_visibility(group_op)
if is_operation_group:
op_group_id = dao.get_operationgroup_by_gid(entity_gid).id
set_group_descendants_visibility(op_group_id)
else:
operation = dao.get_operation_by_gid(entity_gid)
# we assure that if the operation belongs to a group than the visibility will be changed for the entire group
if operation.fk_operation_group is not None:
set_group_descendants_visibility(operation.fk_operation_group)
else:
set_visibility(operation)
def get_operation_details(self, operation_gid, is_group):
"""
:returns: an entity OperationOverlayDetails filled with all information for current operation details.
"""
if is_group:
operation_group = self.get_operation_group_by_gid(operation_gid)
operation = dao.get_operations_in_group(operation_group.id, False, True)
# Reload, to make sure all attributes lazy are populated as well.
operation = dao.get_operation_by_gid(operation.gid)
no_of_op_in_group = dao.get_operations_in_group(operation_group.id, is_count=True)
datatype_group = self.get_datatypegroup_by_op_group_id(operation_group.id)
count_result = dao.count_datatypes_in_group(datatype_group.id)
else:
operation = dao.get_operation_by_gid(operation_gid)
if operation is None:
return None
no_of_op_in_group = 1
count_result = dao.count_resulted_datatypes(operation.id)
user_display_name = dao.get_user_by_id(operation.fk_launched_by).display_name
burst = dao.get_burst_for_operation_id(operation.id)
datatypes_param, all_special_params = self._review_operation_inputs(operation.gid)
op_pid = dao.get_operation_process_for_operation(operation.id)
op_details = OperationOverlayDetails(operation, user_display_name, len(datatypes_param),
count_result, burst, no_of_op_in_group, op_pid)
# Add all parameter which are set differently by the user on this Operation.
if all_special_params is not None:
op_details.add_scientific_fields(all_special_params)
return op_details
@staticmethod
def get_filterable_meta():
"""
Contains all the attributes by which
the user can structure the tree of DataTypes
"""
return DataTypeMetaData.get_filterable_meta()
def get_project_structure(self, project, visibility_filter, first_level, second_level, filter_value):
"""
Find all DataTypes (including the linked ones and the groups) relevant for the current project.
In case of a problem, will return an empty list.
"""
metadata_list = []
dt_list = dao.get_data_in_project(project.id, visibility_filter, filter_value)
for dt in dt_list:
# Prepare the DT results from DB, for usage in controller, by converting into DataTypeMetaData objects
data = {}
is_group = False
group_op = None
dt_entity = dao.get_datatype_by_gid(dt.gid)
if dt_entity is None:
self.logger.warning("Ignored entity (possibly removed DT class)" + str(dt))
continue
# Filter by dt.type, otherwise Links to individual DT inside a group will be mistaken
if dt.type == "DataTypeGroup" and dt.parent_operation.operation_group is not None:
is_group = True
group_op = dt.parent_operation.operation_group
# All these fields are necessary here for dynamic Tree levels.
data[DataTypeMetaData.KEY_DATATYPE_ID] = dt.id
data[DataTypeMetaData.KEY_GID] = dt.gid
data[DataTypeMetaData.KEY_NODE_TYPE] = dt.display_type
data[DataTypeMetaData.KEY_STATE] = dt.state
data[DataTypeMetaData.KEY_SUBJECT] = str(dt.subject)
data[DataTypeMetaData.KEY_TITLE] = dt_entity.display_name
data[DataTypeMetaData.KEY_RELEVANCY] = dt.visible
data[DataTypeMetaData.KEY_LINK] = dt.parent_operation.fk_launched_in != project.id
data[DataTypeMetaData.KEY_TAG_1] = dt.user_tag_1 if dt.user_tag_1 else ''
data[DataTypeMetaData.KEY_TAG_2] = dt.user_tag_2 if dt.user_tag_2 else ''
data[DataTypeMetaData.KEY_TAG_3] = dt.user_tag_3 if dt.user_tag_3 else ''
data[DataTypeMetaData.KEY_TAG_4] = dt.user_tag_4 if dt.user_tag_4 else ''
data[DataTypeMetaData.KEY_TAG_5] = dt.user_tag_5 if dt.user_tag_5 else ''
# Operation related fields:
operation_name = CommonDetails.compute_operation_name(
dt.parent_operation.algorithm.algorithm_category.displayname,
dt.parent_operation.algorithm.displayname)
data[DataTypeMetaData.KEY_OPERATION_TYPE] = operation_name
data[DataTypeMetaData.KEY_OPERATION_ALGORITHM] = dt.parent_operation.algorithm.displayname
data[DataTypeMetaData.KEY_AUTHOR] = dt.parent_operation.user.username
data[DataTypeMetaData.KEY_OPERATION_TAG] = group_op.name if is_group else dt.parent_operation.user_group
data[DataTypeMetaData.KEY_OP_GROUP_ID] = group_op.id if is_group else None
completion_date = dt.parent_operation.completion_date
string_year = completion_date.strftime(MONTH_YEAR_FORMAT) if completion_date is not None else ""
string_month = completion_date.strftime(DAY_MONTH_YEAR_FORMAT) if completion_date is not None else ""
data[DataTypeMetaData.KEY_DATE] = date2string(completion_date) if (completion_date is not None) else ''
data[DataTypeMetaData.KEY_CREATE_DATA_MONTH] = string_year
data[DataTypeMetaData.KEY_CREATE_DATA_DAY] = string_month
data[DataTypeMetaData.KEY_BURST] = dt._parent_burst.name if dt._parent_burst is not None else '-None-'
metadata_list.append(DataTypeMetaData(data, dt.invalid))
return StructureNode.metadata2tree(metadata_list, first_level, second_level, project.id, project.name)
@staticmethod
def get_datatype_details(datatype_gid):
"""
:returns: an array. First entry in array is an instance of DataTypeOverlayDetails\
The second one contains all the possible states for the specified dataType.
"""
meta_atts = DataTypeOverlayDetails()
states = DataTypeMetaData.STATES
try:
datatype_result = dao.get_datatype_details(datatype_gid)
meta_atts.fill_from_datatype(datatype_result, datatype_result._parent_burst)
return meta_atts, states, datatype_result
except Exception:
## We ignore exception here (it was logged above, and we want to return no details).
return meta_atts, states, None
def _remove_project_node_files(self, project_id, gid, skip_validation=False):
"""
Delegate removal of a node in the structure of the project.
In case of a problem will THROW StructureException.
"""
try:
project = self.find_project(project_id)
datatype = dao.get_datatype_by_gid(gid)
links = dao.get_links_for_datatype(datatype.id)
if links:
was_link = False
for link in links:
# This means it's only a link and we need to remove it
if link.fk_from_datatype == datatype.id and link.fk_to_project == project.id:
dao.remove_entity(Links, link.id)
was_link = True
if not was_link:
# Create a clone of the operation
new_op = Operation(dao.get_system_user().id,
links[0].fk_to_project,
datatype.parent_operation.fk_from_algo,
datatype.parent_operation.parameters,
datatype.parent_operation.meta_data,
datatype.parent_operation.status,
datatype.parent_operation.start_date,
datatype.parent_operation.completion_date,
datatype.parent_operation.fk_operation_group,
datatype.parent_operation.additional_info,
datatype.parent_operation.user_group,
datatype.parent_operation.range_values)
new_op = dao.store_entity(new_op)
to_project = self.find_project(links[0].fk_to_project).name
new_op_loaded = dao.get_operation_by_id(new_op.id)
self.structure_helper.write_operation_metadata(new_op_loaded)
full_path = h5.path_for_stored_index(datatype)
self.structure_helper.move_datatype(datatype, to_project, str(new_op.id), full_path)
datatype.fk_from_operation = new_op.id
datatype.parent_operation = new_op
dao.store_entity(datatype)
dao.remove_entity(Links, links[0].id)
else:
specific_remover = get_remover(datatype.type)(datatype)
specific_remover.remove_datatype(skip_validation)
h5_path = h5.path_for_stored_index(datatype)
self.structure_helper.remove_datatype_file(h5_path)
except RemoveDataTypeException:
self.logger.exception("Could not execute operation Node Remove!")
raise
except FileStructureException:
self.logger.exception("Remove operation failed")
raise StructureException("Remove operation failed for unknown reasons.Please contact system administrator.")
def remove_operation(self, operation_id):
"""
Remove a given operation
"""
operation = dao.try_get_operation_by_id(operation_id)
if operation is not None:
self.logger.debug("Deleting operation %s " % operation)
datatypes_for_op = dao.get_results_for_operation(operation_id)
for dt in reversed(datatypes_for_op):
self.remove_datatype(operation.project.id, dt.gid, False)
dao.remove_entity(Operation, operation.id)
self.structure_helper.remove_operation_data(operation.project.name, operation_id)
self.logger.debug("Finished deleting operation %s " % operation)
else:
self.logger.warning("Attempt to delete operation with id=%s which no longer exists." % operation_id)
def remove_datatype(self, project_id, datatype_gid, skip_validation=False):
"""
Method used for removing a dataType. If the given dataType is a DatatypeGroup
or a dataType from a DataTypeGroup than this method will remove the entire group.
The operation(s) used for creating the dataType(s) will also be removed.
"""
datatype = dao.get_datatype_by_gid(datatype_gid)
if datatype is None:
self.logger.warning("Attempt to delete DT[%s] which no longer exists." % datatype_gid)
return
is_datatype_group = False
if dao.is_datatype_group(datatype_gid):
is_datatype_group = True
elif datatype.fk_datatype_group is not None:
is_datatype_group = True
datatype = dao.get_datatype_by_id(datatype.fk_datatype_group)
operations_set = [datatype.fk_from_operation]
correct = True
if is_datatype_group:
self.logger.debug("Removing datatype group %s" % datatype)
data_list = dao.get_datatypes_from_datatype_group(datatype.id)
for adata in data_list:
self._remove_project_node_files(project_id, adata.gid, skip_validation)
if adata.fk_from_operation not in operations_set:
operations_set.append(adata.fk_from_operation)
datatype_group = dao.get_datatype_group_by_gid(datatype.gid)
dao.remove_datatype(datatype_gid)
correct = correct and dao.remove_entity(OperationGroup, datatype_group.fk_operation_group)
else:
self.logger.debug("Removing datatype %s" % datatype)
self._remove_project_node_files(project_id, datatype.gid, skip_validation)
## Remove Operation entity in case no other DataType needs them.
project = dao.get_project_by_id(project_id)
for operation_id in operations_set:
dependent_dt = dao.get_generic_entity(DataType, operation_id, "fk_from_operation")
if len(dependent_dt) > 0:
### Do not remove Operation in case DataType still exist referring it.
continue
correct = correct and dao.remove_entity(Operation, operation_id)
## Make sure Operation folder is removed
self.structure_helper.remove_operation_data(project.name, datatype.fk_from_operation)
if not correct:
raise RemoveDataTypeException("Could not remove DataType " + str(datatype_gid))
def update_metadata(self, submit_data):
"""
Update DataType/ DataTypeGroup metadata
THROW StructureException when input data is invalid.
"""
new_data = dict()
for key in DataTypeOverlayDetails().meta_attributes_list:
if key in submit_data:
new_data[key] = submit_data[key]
if new_data[CommonDetails.CODE_OPERATION_TAG] == '':
new_data[CommonDetails.CODE_OPERATION_TAG] = None
try:
if (CommonDetails.CODE_OPERATION_GROUP_ID in new_data
and new_data[CommonDetails.CODE_OPERATION_GROUP_ID]
and new_data[CommonDetails.CODE_OPERATION_GROUP_ID] != ''):
# We need to edit a group
all_data_in_group = dao.get_datatype_in_group(operation_group_id=
new_data[CommonDetails.CODE_OPERATION_GROUP_ID])
if len(all_data_in_group) < 1:
raise StructureException("Inconsistent group, can not be updated!")
datatype_group = dao.get_generic_entity(DataTypeGroup, all_data_in_group[0].fk_datatype_group)[0]
all_data_in_group.append(datatype_group)
for datatype in all_data_in_group:
new_data[CommonDetails.CODE_GID] = datatype.gid
self._edit_data(datatype, new_data, True)
else:
# Get the required DataType and operation from DB to store changes that will be done in XML.
gid = new_data[CommonDetails.CODE_GID]
datatype = dao.get_datatype_by_gid(gid)
self._edit_data(datatype, new_data)
except Exception as excep:
self.logger.exception(excep)
raise StructureException(str(excep))
def _edit_data(self, datatype, new_data, from_group=False):
"""
Private method, used for editing a meta-data XML file and a DataType row
for a given custom DataType entity with new dictionary of data from UI.
"""
from tvb.basic.traits.types_mapped import MappedType
if isinstance(datatype, MappedType) and not os.path.exists(datatype.get_storage_file_path()):
if not datatype.invalid:
datatype.invalid = True
dao.store_entity(datatype)
return
# 1. First update Operation fields:
# Update group field if possible
new_group_name = new_data[CommonDetails.CODE_OPERATION_TAG]
empty_group_value = (new_group_name is None or new_group_name == "")
if from_group:
if empty_group_value:
raise StructureException("Empty group is not allowed!")
group = dao.get_generic_entity(OperationGroup, new_data[CommonDetails.CODE_OPERATION_GROUP_ID])
if group and len(group) > 0 and new_group_name != group[0].name:
group = group[0]
exists_group = dao.get_generic_entity(OperationGroup, new_group_name, 'name')
if exists_group:
raise StructureException("Group '" + new_group_name + "' already exists.")
group.name = new_group_name
dao.store_entity(group)
else:
operation = dao.get_operation_by_id(datatype.fk_from_operation)
operation.user_group = new_group_name
dao.store_entity(operation)
# 2. Update dateType fields:
datatype.subject = new_data[DataTypeOverlayDetails.DATA_SUBJECT]
datatype.state = new_data[DataTypeOverlayDetails.DATA_STATE]
if DataTypeOverlayDetails.DATA_TAG_1 in new_data:
datatype.user_tag_1 = new_data[DataTypeOverlayDetails.DATA_TAG_1]
if DataTypeOverlayDetails.DATA_TAG_2 in new_data:
datatype.user_tag_2 = new_data[DataTypeOverlayDetails.DATA_TAG_2]
if DataTypeOverlayDetails.DATA_TAG_3 in new_data:
datatype.user_tag_3 = new_data[DataTypeOverlayDetails.DATA_TAG_3]
if DataTypeOverlayDetails.DATA_TAG_4 in new_data:
datatype.user_tag_4 = new_data[DataTypeOverlayDetails.DATA_TAG_4]
if DataTypeOverlayDetails.DATA_TAG_5 in new_data:
datatype.user_tag_5 = new_data[DataTypeOverlayDetails.DATA_TAG_5]
datatype = dao.store_entity(datatype)
# 3. Update MetaData in H5 as well.
datatype.persist_full_metadata()
# 4. Update the group_name/user_group into the operation meta-data file
operation = dao.get_operation_by_id(datatype.fk_from_operation)
self.structure_helper.update_operation_metadata(operation.project.name, new_group_name,
str(datatype.fk_from_operation), from_group)
def get_datatype_and_datatypegroup_inputs_for_operation(self, operation_gid, selected_filter):
"""
Returns the dataTypes that are used as input parameters for the given operation.
'selected_filter' - is expected to be a visibility filter.
If any dataType is part of a dataType group then the dataType group will
be returned instead of that dataType.
"""
all_datatypes = self._review_operation_inputs(operation_gid)[0]
datatype_inputs = []
for datatype in all_datatypes:
if selected_filter.display_name == StaticFiltersFactory.RELEVANT_VIEW:
if datatype.visible:
datatype_inputs.append(datatype)
else:
datatype_inputs.append(datatype)
datatypes = []
datatype_groups = dict()
for data_type in datatype_inputs:
if data_type.fk_datatype_group is None:
datatypes.append(data_type)
elif data_type.fk_datatype_group not in datatype_groups:
dt_group = dao.get_datatype_by_id(data_type.fk_datatype_group)
datatype_groups[data_type.fk_datatype_group] = dt_group
datatypes.extend([v for _, v in six.iteritems(datatype_groups)])
return datatypes
def _review_operation_inputs(self, operation_gid):
"""
:returns: A list of DataTypes that are used as input parameters for the specified operation.
And a dictionary will all operation parameters different then the default ones.
"""
operation = dao.get_operation_by_gid(operation_gid)
parameters = json.loads(operation.parameters)
try:
adapter = ABCAdapter.build_adapter(operation.algorithm)
return adapter.review_operation_inputs(parameters)
except Exception:
self.logger.exception("Could not load details for operation %s" % operation_gid)
inputs_datatypes = []
changed_parameters = dict(Warning="Algorithm changed dramatically. We can not offer more details")
for submit_param in parameters.values():
self.logger.debug("Searching DT by GID %s" % submit_param)
datatype = ABCAdapter.load_entity_by_gid(str(submit_param))
if datatype is not None:
inputs_datatypes.append(datatype)
return inputs_datatypes, changed_parameters
def get_datatypes_inputs_for_operation_group(self, group_id, selected_filter):
"""
Returns the dataType inputs for an operation group. If more dataTypes
are part of the same dataType group then only the dataType group will
be returned instead of them.
"""
operations_gids = dao.get_operations_in_group(group_id, only_gids=True)
op_group_inputs = dict()
for gid in operations_gids:
op_inputs = self.get_datatype_and_datatypegroup_inputs_for_operation(gid[0], selected_filter)
for datatype in op_inputs:
op_group_inputs[datatype.id] = datatype
return list(op_group_inputs.values())
@staticmethod
def get_results_for_operation(operation_id, selected_filter=None):
"""
Retrieve the DataTypes entities resulted after the execution of the given operation.
"""
return dao.get_results_for_operation(operation_id, selected_filter)
@staticmethod
def get_operations_for_datatype_group(datatype_group_id, visibility_filter, only_in_groups=False):
"""
Returns all the operations which uses as an input parameter a dataType from the given DataTypeGroup.
visibility_filter - is a filter used for retrieving all the operations or only the relevant ones.
If only_in_groups is True than this method will return only the operations that are
part from an operation group, otherwise it will return only the operations that
are NOT part of an operation group.
"""
if visibility_filter.display_name != StaticFiltersFactory.RELEVANT_VIEW:
return dao.get_operations_for_datatype_group(datatype_group_id, only_relevant=False,
only_in_groups=only_in_groups)
return dao.get_operations_for_datatype_group(datatype_group_id, only_in_groups=only_in_groups)
@staticmethod
def get_operations_for_datatype(datatype_gid, visibility_filter, only_in_groups=False):
"""
Returns all the operations which uses as an input parameter the dataType with the specified GID.
If only_in_groups is True than this method will return only the operations that are part
from an operation group, otherwise it will return only the operations that are NOT part of an operation group.
"""
if visibility_filter.display_name != StaticFiltersFactory.RELEVANT_VIEW:
return dao.get_operations_for_datatype(datatype_gid, only_relevant=False, only_in_groups=only_in_groups)
return dao.get_operations_for_datatype(datatype_gid, only_in_groups=only_in_groups)
@staticmethod
def get_datatype_by_id(datatype_id):
"""Retrieve a DataType DB reference by its id."""
return dao.get_datatype_by_id(datatype_id)
@staticmethod
def get_datatypegroup_by_gid(datatypegroup_gid):
""" Returns the DataTypeGroup with the specified gid. """
return dao.get_datatype_group_by_gid(datatypegroup_gid)
@staticmethod
def count_datatypes_generated_from(datatype_gid):
"""
A list with all the datatypes resulted from operations that had as
input the datatype given by 'datatype_gid'.
"""
return dao.count_datatypes_generated_from(datatype_gid)
@staticmethod
def get_datatypegroup_by_op_group_id(operation_group_id):
""" Returns the DataTypeGroup with the specified id. """
return dao.get_datatypegroup_by_op_group_id(operation_group_id)
@staticmethod
def get_datatypes_in_project(project_id, only_visible=False):
return dao.get_data_in_project(project_id, only_visible)
@staticmethod
def set_datatype_visibility(datatype_gid, is_visible):
"""
Sets the dataType visibility. If the given dataType is a dataType group or it is part of a
dataType group than this method will set the visibility for each dataType from this group.
"""
def set_visibility(dt):
""" set visibility flag, persist in db and h5"""
dt.visible = is_visible
dt = dao.store_entity(dt)
dt.persist_full_metadata()
def set_group_descendants_visibility(datatype_group_id):
datatypes_in_group = dao.get_datatypes_from_datatype_group(datatype_group_id)
for group_dt in datatypes_in_group:
set_visibility(group_dt)
datatype = dao.get_datatype_by_gid(datatype_gid)
if isinstance(datatype, DataTypeGroup): # datatype is a group
set_group_descendants_visibility(datatype.id)
elif datatype.fk_datatype_group is not None: # datatype is member of a group
set_group_descendants_visibility(datatype.fk_datatype_group)
# the datatype to be updated is the parent datatype group
datatype = dao.get_datatype_by_id(datatype.fk_datatype_group)
# update the datatype or datatype group.
set_visibility(datatype)
@staticmethod
def is_datatype_group(datatype_gid):
""" Used to check if the dataType with the specified GID is a DataTypeGroup. """
return dao.is_datatype_group(datatype_gid)
class ABCAdapter(object):
"""
Root Abstract class for all TVB Adapters.
"""
# todo this constants copy is not nice
TYPE_SELECT = input_tree.TYPE_SELECT
TYPE_MULTIPLE = input_tree.TYPE_MULTIPLE
STATIC_ACCEPTED_TYPES = input_tree.STATIC_ACCEPTED_TYPES
KEY_TYPE = input_tree.KEY_TYPE
KEY_OPTIONS = input_tree.KEY_OPTIONS
KEY_ATTRIBUTES = input_tree.KEY_ATTRIBUTES
KEY_NAME = input_tree.KEY_NAME
KEY_DESCRIPTION = input_tree.KEY_DESCRIPTION
KEY_VALUE = input_tree.KEY_VALUE
KEY_LABEL = input_tree.KEY_LABEL
KEY_DEFAULT = input_tree.KEY_DEFAULT
KEY_DATATYPE = input_tree.KEY_DATATYPE
KEY_DTYPE = input_tree.KEY_DTYPE
KEY_DISABLED = input_tree.KEY_DISABLED
KEY_ALL = input_tree.KEY_ALL
KEY_CONDITION = input_tree.KEY_CONDITION
KEY_FILTERABLE = input_tree.KEY_FILTERABLE
KEY_REQUIRED = input_tree.KEY_REQUIRED
KEY_ID = input_tree.KEY_ID
KEY_UI_HIDE = input_tree.KEY_UI_HIDE
# TODO: move everything related to parameters PRE + POST into parameters_factory
KEYWORD_PARAMS = input_tree.KEYWORD_PARAMS
KEYWORD_SEPARATOR = input_tree.KEYWORD_SEPARATOR
KEYWORD_OPTION = input_tree.KEYWORD_OPTION
INTERFACE_ATTRIBUTES_ONLY = "attributes-only"
INTERFACE_ATTRIBUTES = "attributes"
# model.Algorithm instance that will be set for each adapter created by in build_adapter method
stored_adapter = None
def __init__(self):
# It will be populate with key from DataTypeMetaData
self.meta_data = {
DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT
}
self.generic_attributes = GenericAttributes()
self.generic_attributes.subject = DataTypeMetaData.DEFAULT_SUBJECT
self.file_handler = FilesHelper()
self.storage_path = '.'
# Will be populate with current running operation's identifier
self.operation_id = None
self.user_id = None
self.log = get_logger(self.__class__.__module__)
self.tree_manager = InputTreeManager()
self.submitted_form = None
@classmethod
def get_group_name(cls):
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "name"):
return cls._ui_group.name
return None
@classmethod
def get_group_description(cls):
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "description"):
return cls._ui_group.description
return None
@classmethod
def get_ui_name(cls):
if hasattr(cls, "_ui_name"):
return cls._ui_name
else:
return cls.__name__
@classmethod
def get_ui_description(cls):
if hasattr(cls, "_ui_description"):
return cls._ui_description
@classmethod
def get_ui_subsection(cls):
if hasattr(cls, "_ui_subsection"):
return cls._ui_subsection
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "subsection"):
return cls._ui_group.subsection
@staticmethod
def can_be_active():
"""
To be overridden where needed (e.g. Matlab dependent adapters).
:return: By default True, and False when the current Adapter can not be executed in the current env
for various reasons (e.g. no Matlab or Octave installed)
"""
return True
def get_input_tree(self):
"""
Describes inputs and outputs of the launch method.
"""
return None
def submit_form(self, form):
self.submitted_form = form
# TODO separate usage of get_form_class (returning a class) and return of a submitted instance
def get_form(self):
if self.submitted_form is not None:
return self.submitted_form
return self.get_form_class()
@abstractmethod
def get_form_class(self):
return None
@abstractmethod
def get_output(self):
"""
Describes inputs and outputs of the launch method.
"""
def configure(self, **kwargs):
"""
To be implemented in each Adapter that requires any specific configurations
before the actual launch.
"""
@abstractmethod
def get_required_memory_size(self, **kwargs):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter.
"""
@abstractmethod
def get_required_disk_size(self, **kwargs):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter in kilo-Bytes.
"""
def get_execution_time_approximation(self, **kwargs):
"""
Method should approximate based on input arguments, the time it will take for the operation
to finish (in seconds).
"""
return -1
@abstractmethod
def launch(self):
"""
To be implemented in each Adapter.
Will contain the logic of the Adapter.
Any returned DataType will be stored in DB, by the Framework.
"""
def add_operation_additional_info(self, message):
"""
Adds additional info on the operation to be displayed in the UI. Usually a warning message.
"""
current_op = dao.get_operation_by_id(self.operation_id)
current_op.additional_info = message
dao.store_entity(current_op)
def _prepare_generic_attributes(self, user_tag=None):
self.generic_attributes.subject = str(
self.meta_data.get(DataTypeMetaData.KEY_SUBJECT))
self.generic_attributes.state = self.meta_data.get(
DataTypeMetaData.KEY_STATE)
perpetuated_identifier = self.generic_attributes.user_tag_1
if DataTypeMetaData.KEY_TAG_1 in self.meta_data:
perpetuated_identifier = self.meta_data.get(
DataTypeMetaData.KEY_TAG_1)
if not self.generic_attributes.user_tag_1:
self.generic_attributes.user_tag_1 = user_tag if user_tag is not None else perpetuated_identifier
else:
self.generic_attributes.user_tag_2 = user_tag if user_tag is not None else perpetuated_identifier
@nan_not_allowed()
def _prelaunch(self,
operation,
uid=None,
available_disk_space=0,
**kwargs):
"""
Method to wrap LAUNCH.
Will prepare data, and store results on return.
"""
self.meta_data.update(json.loads(operation.meta_data))
self.storage_path = self.file_handler.get_project_folder(
operation.project, str(operation.id))
self.operation_id = operation.id
self.current_project_id = operation.project.id
self.user_id = operation.fk_launched_by
self.configure(**kwargs)
# Compare the amount of memory the current algorithms states it needs,
# with the average between the RAM available on the OS and the free memory at the current moment.
# We do not consider only the free memory, because some OSs are freeing late and on-demand only.
total_free_memory = psutil.virtual_memory().free + psutil.swap_memory(
).free
total_existent_memory = psutil.virtual_memory(
).total + psutil.swap_memory().total
memory_reference = (total_free_memory + total_existent_memory) / 2
adapter_required_memory = self.get_required_memory_size(**kwargs)
if adapter_required_memory > memory_reference:
msg = "Machine does not have enough RAM memory for the operation (expected %.2g GB, but found %.2g GB)."
raise NoMemoryAvailableException(
msg %
(adapter_required_memory / 2**30, memory_reference / 2**30))
# Compare the expected size of the operation results with the HDD space currently available for the user
# TVB defines a quota per user.
required_disk_space = self.get_required_disk_size(**kwargs)
if available_disk_space < 0:
msg = "You have exceeded you HDD space quota by %.2f MB Stopping execution."
raise NoMemoryAvailableException(msg %
(-available_disk_space / 2**10))
if available_disk_space < required_disk_space:
msg = (
"You only have %.2f GB of disk space available but the operation you "
"launched might require %.2f Stopping execution...")
raise NoMemoryAvailableException(
msg %
(available_disk_space / 2**20, required_disk_space / 2**20))
operation.start_now()
operation.estimated_disk_size = required_disk_space
dao.store_entity(operation)
self._prepare_generic_attributes(uid)
result = self.launch(**kwargs)
if not isinstance(result, (list, tuple)):
result = [
result,
]
self.__check_integrity(result)
return self._capture_operation_results(result)
def _capture_operation_results(self, result):
"""
After an operation was finished, make sure the results are stored
in DB storage and the correct meta-data,IDs are set.
"""
data_type_group_id = None
operation = dao.get_operation_by_id(self.operation_id)
if operation.user_group is None or len(operation.user_group) == 0:
operation.user_group = date2string(
datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT)
operation = dao.store_entity(operation)
if self._is_group_launch():
data_type_group_id = dao.get_datatypegroup_by_op_group_id(
operation.fk_operation_group).id
burst_reference = None
if DataTypeMetaData.KEY_BURST in self.meta_data:
burst_reference = self.meta_data[DataTypeMetaData.KEY_BURST]
count_stored = 0
group_type = None # In case of a group, the first not-none type is sufficient to memorize here
for res in result:
if res is None:
continue
res.subject = self.generic_attributes.subject
res.state = self.generic_attributes.state
res.fk_parent_burst = burst_reference
res.fk_from_operation = self.operation_id
res.framework_metadata = self.meta_data
res.user_tag_1 = self.generic_attributes.user_tag_1
res.user_tag_2 = self.generic_attributes.user_tag_2
res.fk_datatype_group = data_type_group_id
# Compute size-on disk, in case file-storage is used
associated_file = h5.path_for_stored_index(res)
if os.path.exists(associated_file):
res.disk_size = self.file_handler.compute_size_on_disk(
associated_file)
with H5File.from_file(associated_file) as f:
f.store_generic_attributes(self.generic_attributes)
dao.store_entity(res)
group_type = res.type
count_stored += 1
if count_stored > 0 and self._is_group_launch():
# Update the operation group name
operation_group = dao.get_operationgroup_by_id(
operation.fk_operation_group)
operation_group.fill_operationgroup_name(group_type)
dao.store_entity(operation_group)
return 'Operation ' + str(
self.operation_id) + ' has finished.', count_stored
def __check_integrity(self, result):
"""
Check that the returned parameters for LAUNCH operation
are of the type specified in the adapter's interface.
"""
for result_entity in result:
if result_entity is None:
continue
if not self.__is_data_in_supported_types(result_entity):
msg = "Unexpected output DataType %s"
raise InvalidParameterException(msg % type(result_entity))
def __is_data_in_supported_types(self, data):
if data is None:
return True
for supported_type in self.get_output():
if isinstance(data, supported_type):
return True
# Data can't be mapped on any supported type !!
return False
def _is_group_launch(self):
"""
Return true if this adapter is launched from a group of operations
"""
operation = dao.get_operation_by_id(self.operation_id)
return operation.fk_operation_group is not None
@staticmethod
def load_entity_by_gid(data_gid):
"""
Load a generic DataType, specified by GID.
"""
return load_entity_by_gid(data_gid)
@staticmethod
def build_adapter_from_class(adapter_class):
"""
Having a subclass of ABCAdapter, prepare an instance for launching an operation with it.
"""
if not issubclass(adapter_class, ABCAdapter):
raise IntrospectionException(
"Invalid data type: It should extend adapters.ABCAdapter!")
try:
stored_adapter = dao.get_algorithm_by_module(
adapter_class.__module__, adapter_class.__name__)
adapter_instance = adapter_class()
adapter_instance.stored_adapter = stored_adapter
return adapter_instance
except Exception as excep:
LOGGER.exception(excep)
raise IntrospectionException(str(excep))
@staticmethod
def build_adapter(stored_adapter):
"""
Having a module and a class name, create an instance of ABCAdapter.
"""
try:
ad_module = importlib.import_module(stored_adapter.module)
adapter_class = getattr(ad_module, stored_adapter.classname)
adapter_instance = adapter_class()
adapter_instance.stored_adapter = stored_adapter
return adapter_instance
except Exception:
msg = "Could not load Adapter Instance for Stored row %s" % stored_adapter
LOGGER.exception(msg)
raise IntrospectionException(msg)
# METHODS for PROCESSING PARAMETERS start here #############################
def review_operation_inputs(self, parameters):
"""
:returns: a list with the inputs from the parameters list that are instances of DataType,\
and a dictionary with all parameters which are different than the declared defauts
"""
flat_interface = self.flaten_input_interface()
return self.tree_manager.review_operation_inputs(
parameters, flat_interface)
def prepare_ui_inputs(self, kwargs, validation_required=True):
"""
Prepare the inputs received from a HTTP Post in a form that will be
used by the Python adapter.
"""
algorithm_inputs = self.get_input_tree()
algorithm_inputs = InputTreeManager.prepare_param_names(
algorithm_inputs)
self.tree_manager.append_required_defaults(kwargs, algorithm_inputs)
return self.convert_ui_inputs(kwargs,
validation_required=validation_required)
def convert_ui_inputs(self, kwargs, validation_required=True):
"""
Convert HTTP POST parameters into Python parameters.
"""
return self.tree_manager.convert_ui_inputs(
self.flaten_input_interface(), kwargs, self.meta_data,
validation_required)
def noise_configurable_parameters(self):
return [
entry[self.KEY_NAME] for entry in self.flaten_input_interface()
if 'configurableNoise' in entry
]
def flaten_input_interface(self):
# TODO: temporary condition to pass introspection on neoforms
form = self.get_form_class()()
if form:
return [
form._get_original_field_name(form_field)
for form_field in form.fields
]
return self.tree_manager.flatten(self.get_input_tree())
class TestSimulationResource(RestResourceTest):
def transactional_setup_method(self):
self.test_user = TestFactory.create_user('Rest_User')
self.test_project = TestFactory.create_project(
self.test_user, 'Rest_Project', users=[self.test_user.id])
self.simulation_resource = FireSimulationResource()
self.files_helper = FilesHelper()
def test_server_fire_simulation_inexistent_gid(self, mocker):
self._mock_user(mocker)
project_gid = "inexistent-gid"
dummy_file = FileStorage(BytesIO(b"test"), 'test.zip')
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request')
request_mock.files = {
RequestFileKey.SIMULATION_FILE_KEY.value: dummy_file
}
with pytest.raises(InvalidIdentifierException):
self.simulation_resource.post(project_gid=project_gid)
def test_server_fire_simulation_no_file(self, mocker):
self._mock_user(mocker)
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request')
request_mock.files = {}
with pytest.raises(InvalidIdentifierException):
self.simulation_resource.post(project_gid='')
def test_server_fire_simulation_bad_extension(self, mocker):
self._mock_user(mocker)
dummy_file = FileStorage(BytesIO(b"test"), 'test.txt')
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request')
request_mock.files = {
RequestFileKey.SIMULATION_FILE_KEY.value: dummy_file
}
with pytest.raises(InvalidIdentifierException):
self.simulation_resource.post(project_gid='')
def test_server_fire_simulation(self, mocker, connectivity_factory):
self._mock_user(mocker)
input_folder = self.files_helper.get_project_folder(self.test_project)
sim_dir = os.path.join(input_folder, 'test_sim')
if not os.path.isdir(sim_dir):
os.makedirs(sim_dir)
simulator = SimulatorAdapterModel()
simulator.connectivity = connectivity_factory().gid
h5.store_view_model(simulator, sim_dir)
zip_filename = os.path.join(input_folder,
RequestFileKey.SIMULATION_FILE_NAME.value)
FilesHelper().zip_folder(zip_filename, sim_dir)
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request')
fp = open(zip_filename, 'rb')
request_mock.files = {
RequestFileKey.SIMULATION_FILE_KEY.value:
FileStorage(fp, os.path.basename(zip_filename))
}
def launch_sim(self, user_id, project, algorithm, zip_folder_path,
simulator_file):
return Operation('', '', '', {})
# Mock simulation launch and current user
mocker.patch.object(SimulatorService, 'prepare_simulation_on_server',
launch_sim)
operation_gid, status = self.simulation_resource.post(
project_gid=self.test_project.gid)
fp.close()
assert type(operation_gid) is str
assert status == 201
def transactional_teardown_method(self):
self.files_helper.remove_project_structure(self.test_project.name)
class FlowService:
"""
Service Layer for all TVB generic Work-Flow operations.
"""
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.file_helper = FilesHelper()
self.input_tree_manager = InputTreeManager()
def get_category_by_id(self, identifier):
""" Pass to DAO the retrieve of category by ID operation."""
return dao.get_category_by_id(identifier)
@staticmethod
def get_raw_categories():
""":returns: AlgorithmCategory list of entities that have results in RAW state (Creators/Uploaders)"""
return dao.get_raw_categories()
@staticmethod
def get_visualisers_category():
"""Retrieve all Algorithm categories, with display capability"""
result = dao.get_visualisers_categories()
if not result:
raise ValueError("View Category not found!!!")
return result[0]
@staticmethod
def get_algorithm_by_identifier(ident):
"""
Retrieve Algorithm entity by ID.
Return None, if ID is not found in DB.
"""
return dao.get_algorithm_by_id(ident)
@staticmethod
def load_operation(operation_id):
""" Retrieve previously stored Operation from DB, and load operation.burst attribute"""
operation = dao.get_operation_by_id(operation_id)
operation.burst = dao.get_burst_for_operation_id(operation_id)
return operation
@staticmethod
def get_operation_numbers(proj_id):
""" Count total number of operations started for current project. """
return dao.get_operation_numbers(proj_id)
def prepare_adapter(self, project_id, stored_adapter):
"""
Having a StoredAdapter, return the Tree Adapter Interface object, populated with datatypes from 'project_id'.
"""
adapter_module = stored_adapter.module
adapter_name = stored_adapter.classname
try:
# Prepare Adapter Interface, by populating with existent data,
# in case of a parameter of type DataType.
adapter_instance = ABCAdapter.build_adapter(stored_adapter)
interface = adapter_instance.get_input_tree()
interface = self.input_tree_manager.fill_input_tree_with_options(
interface, project_id, stored_adapter.fk_category)
interface = self.input_tree_manager.prepare_param_names(interface)
return interface
except Exception:
self.logger.exception('Not found:' + adapter_name + ' in:' +
adapter_module)
raise OperationException("Could not prepare " + adapter_name)
@staticmethod
def get_algorithm_by_module_and_class(module, classname):
"""
Get the db entry from the algorithm table for the given module and
class.
"""
return dao.get_algorithm_by_module(module, classname)
@staticmethod
def get_available_datatypes(project_id, data_type_cls, filters=None):
"""
Return all dataTypes that match a given name and some filters.
:param data_type_cls: either a fully qualified class name or a class object
"""
return get_filtered_datatypes(project_id, data_type_cls, filters)
@staticmethod
def create_link(data_ids, project_id):
"""
For a list of dataType IDs and a project id create all the required links.
"""
for data in data_ids:
link = model.Links(data, project_id)
dao.store_entity(link)
@staticmethod
def remove_link(dt_id, project_id):
"""
Remove the link from the datatype given by dt_id to project given by project_id.
"""
link = dao.get_link(dt_id, project_id)
if link is not None:
dao.remove_entity(model.Links, link.id)
def fire_operation(self,
adapter_instance,
current_user,
project_id,
visible=True,
**data):
"""
Launch an operation, specified by AdapterInstance, for CurrentUser,
Current Project and a given set of UI Input Data.
"""
operation_name = str(adapter_instance.__class__.__name__)
try:
self.logger.info("Starting operation " + operation_name)
project = dao.get_project_by_id(project_id)
tmp_folder = self.file_helper.get_project_folder(
project, self.file_helper.TEMP_FOLDER)
result = OperationService().initiate_operation(
current_user, project.id, adapter_instance, tmp_folder,
visible, **data)
self.logger.info("Finished operation:" + operation_name)
return result
except TVBException, excep:
self.logger.exception(
"Could not launch operation " + operation_name +
" with the given set of input data, because: " + excep.message)
raise OperationException(excep.message, excep)
except Exception, excep:
self.logger.exception("Could not launch operation " +
operation_name +
" with the given set of input data!")
raise OperationException(str(excep))
class ABCAdapter(object):
"""
Root Abstract class for all TVB Adapters.
"""
# model.Algorithm instance that will be set for each adapter class created by in build_adapter method
stored_adapter = None
launch_mode = AdapterLaunchModeEnum.ASYNC_DIFF_MEM
def __init__(self):
self.generic_attributes = GenericAttributes()
self.generic_attributes.subject = DataTypeMetaData.DEFAULT_SUBJECT
self.file_handler = FilesHelper()
self.storage_path = '.'
# Will be populate with current running operation's identifier
self.operation_id = None
self.user_id = None
self.submitted_form = None
self.log = get_logger(self.__class__.__module__)
@classmethod
def get_group_name(cls):
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "name"):
return cls._ui_group.name
return None
@classmethod
def get_group_description(cls):
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "description"):
return cls._ui_group.description
return None
@classmethod
def get_ui_name(cls):
if hasattr(cls, "_ui_name"):
return cls._ui_name
else:
return cls.__name__
@classmethod
def get_ui_description(cls):
if hasattr(cls, "_ui_description"):
return cls._ui_description
@classmethod
def get_ui_subsection(cls):
if hasattr(cls, "_ui_subsection"):
return cls._ui_subsection
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "subsection"):
return cls._ui_group.subsection
@staticmethod
def can_be_active():
"""
To be overridden where needed (e.g. Matlab dependent adapters).
:return: By default True, and False when the current Adapter can not be executed in the current env
for various reasons (e.g. no Matlab or Octave installed)
"""
return True
def submit_form(self, form):
self.submitted_form = form
# TODO separate usage of get_form_class (returning a class) and return of a submitted instance
def get_form(self):
if self.submitted_form is not None:
return self.submitted_form
return self.get_form_class()
@abstractmethod
def get_form_class(self):
return None
def get_adapter_fragments(self, view_model):
"""
The result will be used for introspecting and checking operation changed input
params from the defaults, to show in web gui.
:return: a list of ABCAdapterForm classes, in case the current Adapter GUI
will be composed of multiple sub-forms.
"""
return {}
def get_view_model_class(self):
return self.get_form_class().get_view_model()
@abstractmethod
def get_output(self):
"""
Describes inputs and outputs of the launch method.
"""
def configure(self, view_model):
"""
To be implemented in each Adapter that requires any specific configurations
before the actual launch.
"""
@abstractmethod
def get_required_memory_size(self, view_model):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter.
"""
@abstractmethod
def get_required_disk_size(self, view_model):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter in kilo-Bytes.
"""
def get_execution_time_approximation(self, view_model):
"""
Method should approximate based on input arguments, the time it will take for the operation
to finish (in seconds).
"""
return -1
@abstractmethod
def launch(self, view_model):
"""
To be implemented in each Adapter.
Will contain the logic of the Adapter.
Takes a ViewModel with data, dependency direction is: Adapter -> Form -> ViewModel
Any returned DataType will be stored in DB, by the Framework.
:param view_model: the data model corresponding to the current adapter
"""
def add_operation_additional_info(self, message):
"""
Adds additional info on the operation to be displayed in the UI. Usually a warning message.
"""
current_op = dao.get_operation_by_id(self.operation_id)
current_op.additional_info = message
dao.store_entity(current_op)
def extract_operation_data(self, operation):
operation = dao.get_operation_by_id(operation.id)
project = dao.get_project_by_id(operation.fk_launched_in)
self.storage_path = self.file_handler.get_project_folder(project, str(operation.id))
self.operation_id = operation.id
self.current_project_id = operation.project.id
self.user_id = operation.fk_launched_by
def _ensure_enough_resources(self, available_disk_space, view_model):
# Compare the amount of memory the current algorithms states it needs,
# with the average between the RAM available on the OS and the free memory at the current moment.
# We do not consider only the free memory, because some OSs are freeing late and on-demand only.
total_free_memory = psutil.virtual_memory().free + psutil.swap_memory().free
total_existent_memory = psutil.virtual_memory().total + psutil.swap_memory().total
memory_reference = (total_free_memory + total_existent_memory) / 2
adapter_required_memory = self.get_required_memory_size(view_model)
if adapter_required_memory > memory_reference:
msg = "Machine does not have enough RAM memory for the operation (expected %.2g GB, but found %.2g GB)."
raise NoMemoryAvailableException(msg % (adapter_required_memory / 2 ** 30, memory_reference / 2 ** 30))
# Compare the expected size of the operation results with the HDD space currently available for the user
# TVB defines a quota per user.
required_disk_space = self.get_required_disk_size(view_model)
if available_disk_space < 0:
msg = "You have exceeded you HDD space quota by %.2f MB Stopping execution."
raise NoMemoryAvailableException(msg % (- available_disk_space / 2 ** 10))
if available_disk_space < required_disk_space:
msg = ("You only have %.2f GB of disk space available but the operation you "
"launched might require %.2f Stopping execution...")
raise NoMemoryAvailableException(msg % (available_disk_space / 2 ** 20, required_disk_space / 2 ** 20))
return required_disk_space
def _update_operation_entity(self, operation, required_disk_space):
operation.start_now()
operation.estimated_disk_size = required_disk_space
dao.store_entity(operation)
@nan_not_allowed()
def _prelaunch(self, operation, view_model, uid=None, available_disk_space=0):
"""
Method to wrap LAUNCH.
Will prepare data, and store results on return.
"""
self.extract_operation_data(operation)
self.generic_attributes.fill_from(view_model.generic_attributes)
self.configure(view_model)
required_disk_size = self._ensure_enough_resources(available_disk_space, view_model)
self._update_operation_entity(operation, required_disk_size)
if not self.generic_attributes.user_tag_1:
self.generic_attributes.user_tag_1 = uid
else:
self.generic_attributes.user_tag_2 = uid
result = self.launch(view_model)
if not isinstance(result, (list, tuple)):
result = [result, ]
self.__check_integrity(result)
return self._capture_operation_results(result)
def _capture_operation_results(self, result):
"""
After an operation was finished, make sure the results are stored
in DB storage and the correct meta-data,IDs are set.
"""
data_type_group_id = None
operation = dao.get_operation_by_id(self.operation_id)
if operation.user_group is None or len(operation.user_group) == 0:
operation.user_group = date2string(datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT)
operation = dao.store_entity(operation)
if self._is_group_launch():
data_type_group_id = dao.get_datatypegroup_by_op_group_id(operation.fk_operation_group).id
count_stored = 0
group_type = None # In case of a group, the first not-none type is sufficient to memorize here
for res in result:
if res is None:
continue
if not res.fixed_generic_attributes:
res.fill_from_generic_attributes(self.generic_attributes)
res.fk_from_operation = self.operation_id
res.fk_datatype_group = data_type_group_id
associated_file = h5.path_for_stored_index(res)
if os.path.exists(associated_file):
if not res.fixed_generic_attributes:
with H5File.from_file(associated_file) as f:
f.store_generic_attributes(self.generic_attributes)
# Compute size-on disk, in case file-storage is used
res.disk_size = self.file_handler.compute_size_on_disk(associated_file)
dao.store_entity(res)
res.after_store()
group_type = res.type
count_stored += 1
if count_stored > 0 and self._is_group_launch():
# Update the operation group name
operation_group = dao.get_operationgroup_by_id(operation.fk_operation_group)
operation_group.fill_operationgroup_name(group_type)
dao.store_entity(operation_group)
return 'Operation ' + str(self.operation_id) + ' has finished.', count_stored
def __check_integrity(self, result):
"""
Check that the returned parameters for LAUNCH operation
are of the type specified in the adapter's interface.
"""
for result_entity in result:
if result_entity is None:
continue
if not self.__is_data_in_supported_types(result_entity):
msg = "Unexpected output DataType %s"
raise InvalidParameterException(msg % type(result_entity))
def __is_data_in_supported_types(self, data):
if data is None:
return True
for supported_type in self.get_output():
if isinstance(data, supported_type):
return True
# Data can't be mapped on any supported type !!
return False
def _is_group_launch(self):
"""
Return true if this adapter is launched from a group of operations
"""
operation = dao.get_operation_by_id(self.operation_id)
return operation.fk_operation_group is not None
def _get_output_path(self):
return self.storage_path
def load_entity_by_gid(self, data_gid):
# type: (typing.Union[uuid.UUID, str]) -> DataType
"""
Load a generic DataType, specified by GID.
"""
idx = load_entity_by_gid(data_gid)
if idx and self.generic_attributes.parent_burst is None:
# Only in case the BurstConfiguration references hasn't been set already, take it from the current DT
self.generic_attributes.parent_burst = idx.fk_parent_burst
return idx
def load_traited_by_gid(self, data_gid):
# type: (typing.Union[uuid.UUID, str]) -> HasTraits
"""
Load a generic HasTraits instance, specified by GID.
"""
index = self.load_entity_by_gid(data_gid)
return h5.load_from_index(index)
def load_with_references(self, dt_gid):
# type: (typing.Union[uuid.UUID, str]) -> HasTraits
dt_index = self.load_entity_by_gid(dt_gid)
h5_path = h5.path_for_stored_index(dt_index)
dt, _ = h5.load_with_references(h5_path)
return dt
def view_model_to_has_traits(self, view_model):
# type: (ViewModel) -> HasTraits
has_traits_class = view_model.linked_has_traits
has_traits = has_traits_class()
view_model_class = type(view_model)
if not has_traits_class:
raise Exception("There is no linked HasTraits for this ViewModel {}".format(type(view_model)))
for attr_name in has_traits_class.declarative_attrs:
view_model_class_attr = getattr(view_model_class, attr_name)
view_model_attr = getattr(view_model, attr_name)
if isinstance(view_model_class_attr, DataTypeGidAttr) and view_model_attr:
attr_value = self.load_with_references(view_model_attr)
elif isinstance(view_model_class_attr, Attr) and isinstance(view_model_attr, ViewModel):
attr_value = self.view_model_to_has_traits(view_model_attr)
elif isinstance(view_model_class_attr, List) and len(view_model_attr) > 0 and isinstance(view_model_attr[0],
ViewModel):
attr_value = list()
for view_model_elem in view_model_attr:
elem = self.view_model_to_has_traits(view_model_elem)
attr_value.append(elem)
else:
attr_value = view_model_attr
setattr(has_traits, attr_name, attr_value)
return has_traits
@staticmethod
def build_adapter_from_class(adapter_class):
"""
Having a subclass of ABCAdapter, prepare an instance for launching an operation with it.
"""
if not issubclass(adapter_class, ABCAdapter):
raise IntrospectionException("Invalid data type: It should extend adapters.ABCAdapter!")
try:
stored_adapter = dao.get_algorithm_by_module(adapter_class.__module__, adapter_class.__name__)
adapter_instance = adapter_class()
adapter_instance.stored_adapter = stored_adapter
return adapter_instance
except Exception as excep:
LOGGER.exception(excep)
raise IntrospectionException(str(excep))
@staticmethod
def determine_adapter_class(stored_adapter):
# type: (Algorithm) -> ABCAdapter
"""
Determine the class of an adapter based on module and classname strings from stored_adapter
:param stored_adapter: Algorithm or AlgorithmDTO type
:return: a subclass of ABCAdapter
"""
ad_module = importlib.import_module(stored_adapter.module)
adapter_class = getattr(ad_module, stored_adapter.classname)
return adapter_class
@staticmethod
def build_adapter(stored_adapter):
# type: (Algorithm) -> ABCAdapter
"""
Having a module and a class name, create an instance of ABCAdapter.
"""
try:
adapter_class = ABCAdapter.determine_adapter_class(stored_adapter)
adapter_instance = adapter_class()
adapter_instance.stored_adapter = stored_adapter
return adapter_instance
except Exception:
msg = "Could not load Adapter Instance for Stored row %s" % stored_adapter
LOGGER.exception(msg)
raise IntrospectionException(msg)
def load_view_model(self, operation):
storage_path = self.file_handler.get_project_folder(operation.project, str(operation.id))
input_gid = operation.view_model_gid
return h5.load_view_model(input_gid, storage_path)
def array_size2kb(self, size):
"""
:param size: size in bytes
:return: size in kB
"""
return size * TvbProfile.current.MAGIC_NUMBER / 8 / 2 ** 10
class DiagnoseDiskUsage(object):
FORMAT_DT = ' {:14} {:20} {:>12} {:>12} {:>12} {:>12}'
HEADER_DT = FORMAT_DT.format('', '', 'disk_size(kib)', 'db_size(kib)', 'delta(kib)', 'ratio(%)')
def __init__(self, prj_id):
self.file_helper = FilesHelper()
self.expected_files = set()
self.prj_disk_size, self.prj_db_size = 0, 0
try:
dao.session.open_session()
# We do not fetch the project using dao because dao will detach it from the session.
# We want to query on the fly on attribute access and this requires attached objects.
# This code is doing a tree traversal of the db.
# The query on attribute access style fits better than aggregating queries.
self.project = dao.session.query(Project).filter(Project.id == prj_id).one()
self.expected_files.add(self.file_helper.get_project_meta_file_path(self.project.name))
root_path = self.file_helper.get_project_folder(self.project)
print()
print('Reporting disk for project {} in {}'.format(self.project.name, root_path))
print()
print(self.HEADER_DT)
for op in self.project.OPERATIONS:
self.analyse_operation(op)
print(self.HEADER_DT)
self.print_usage_line('Project', 'total', self.prj_disk_size, self.prj_db_size)
print()
self.list_unexpected_project_files(root_path)
print()
finally:
dao.session.close_session()
@staticmethod
def get_h5_by_gid(root, gid):
for f in os.listdir(root):
fp = os.path.join(root, f)
if gid in f and os.path.isfile(fp):
return fp
@staticmethod
def get_file_kib_size(fp):
return int(round((os.path.getsize(fp) / 1024.)))
@staticmethod
def print_usage_line(col1, col2, actual, expected):
if expected != 0:
ratio = int(100.0 * actual / expected)
if ratio > 200:
ratio = "! %s" % ratio
else:
ratio = str(ratio)
else:
ratio = 'inf'
delta = actual - expected
if delta > 100:
delta = "! %s" % delta
else:
delta = str(delta)
print(DiagnoseDiskUsage.FORMAT_DT.format(col1, col2, '{:,}'.format(actual),
'{:,}'.format(expected), delta, ratio))
def analyse_operation(self, op):
op_disk_size, op_db_size = 0, 0
print('Operation {} : {}'.format(op.id, op.algorithm.name))
for dt in op.DATA_TYPES:
if dt.type == 'DataTypeGroup':
# these have no h5
continue
op_pth = self.file_helper.get_operation_folder(self.project.name, op.id)
dt_pth = self.get_h5_by_gid(op_pth, dt.gid)
dt_actual_disk_size = self.get_file_kib_size(dt_pth)
db_disk_size = dt.disk_size or 0
op_disk_size += dt_actual_disk_size
op_db_size += db_disk_size
self.print_usage_line(dt.gid[:12], dt.type, dt_actual_disk_size, db_disk_size)
self.expected_files.add(dt_pth)
self.prj_disk_size += op_disk_size
self.prj_db_size += op_db_size
self.print_usage_line('', 'total :', op_disk_size, op_db_size)
print()
def list_unexpected_project_files(self, root_path):
unexpected = []
for r, d, files in os.walk(root_path):
for f in files:
pth = os.path.join(r, f)
if pth not in self.expected_files:
unexpected.append(pth)
print('Unexpected project files :')
for f in unexpected:
print(f)
if not unexpected:
print('yey! none found')
class FilesHelperTest(TransactionalTestCase):
"""
This class contains tests for the tvb.core.entities.file.files_helper module.
"""
PROJECT_NAME = "test_proj"
def setUp(self):
"""
Set up the context needed by the tests.
"""
self.files_helper = FilesHelper()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user, self.PROJECT_NAME)
def tearDown(self):
""" Remove generated project during tests. """
self.delete_project_folders()
def test_check_created(self):
""" Test standard flows for check created. """
self.files_helper.check_created()
self.assertTrue(os.path.exists(root_storage), "Storage not created!")
self.files_helper.check_created(os.path.join(root_storage, "test"))
self.assertTrue(os.path.exists(root_storage), "Storage not created!")
self.assertTrue(os.path.exists(os.path.join(root_storage, "test")), "Test directory not created!")
def test_get_project_folder(self):
"""
Test the get_project_folder method which should create a folder in case
it doesn't already exist.
"""
project_path = self.files_helper.get_project_folder(self.test_project)
self.assertTrue(os.path.exists(project_path), "Folder doesn't exist")
folder_path = self.files_helper.get_project_folder(self.test_project, "43")
self.assertTrue(os.path.exists(project_path), "Folder doesn't exist")
self.assertTrue(os.path.exists(folder_path), "Folder doesn't exist")
def test_rename_project_structure(self):
""" Try to rename the folder structure of a project. Standard flow. """
self.files_helper.get_project_folder(self.test_project)
path, name = self.files_helper.rename_project_structure(self.test_project.name, "new_name")
self.assertNotEqual(path, name, "Rename didn't take effect.")
def test_rename_structure_same_name(self):
""" Try to rename the folder structure of a project. Same name. """
self.files_helper.get_project_folder(self.test_project)
self.assertRaises(FileStructureException, self.files_helper.rename_project_structure,
self.test_project.name, self.PROJECT_NAME)
def test_remove_project_structure(self):
""" Check that remove project structure deletes the corresponding folder. Standard flow. """
full_path = self.files_helper.get_project_folder(self.test_project)
self.assertTrue(os.path.exists(full_path), "Folder was not created.")
self.files_helper.remove_project_structure(self.test_project.name)
self.assertFalse(os.path.exists(full_path), "Project folder not deleted.")
def test_write_project_metadata(self):
""" Write XML for test-project. """
self.files_helper.write_project_metadata(self.test_project)
expected_file = self.files_helper.get_project_meta_file_path(self.PROJECT_NAME)
self.assertTrue(os.path.exists(expected_file))
project_meta = XMLReader(expected_file).read_metadata()
loaded_project = model.Project(None, None)
loaded_project.from_dict(project_meta, self.test_user.id)
self.assertEqual(self.test_project.name, loaded_project.name)
self.assertEqual(self.test_project.description, loaded_project.description)
self.assertEqual(self.test_project.gid, loaded_project.gid)
expected_dict = self.test_project.to_dict()[1]
del expected_dict['last_updated']
found_dict = loaded_project.to_dict()[1]
del found_dict['last_updated']
self.assertDictContainsSubset(expected_dict, found_dict)
self.assertDictContainsSubset(found_dict, expected_dict)
def test_write_operation_metadata(self):
"""
Test that a correct XML is created for an operation.
"""
operation = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)
expected_file = self.files_helper.get_operation_meta_file_path(self.PROJECT_NAME, operation.id)
self.assertFalse(os.path.exists(expected_file))
self.files_helper.write_operation_metadata(operation)
self.assertTrue(os.path.exists(expected_file))
operation_meta = XMLReader(expected_file).read_metadata()
loaded_operation = model.Operation(None, None, None, None)
loaded_operation.from_dict(operation_meta, dao)
expected_dict = operation.to_dict()[1]
found_dict = loaded_operation.to_dict()[1]
for key, value in expected_dict.iteritems():
self.assertEqual(str(value), str(found_dict[key]))
# Now validate that operation metaData can be also updated
self.assertNotEqual("new_group_name", found_dict['user_group'])
self.files_helper.update_operation_metadata(self.PROJECT_NAME, "new_group_name", operation.id)
found_dict = XMLReader(expected_file).read_metadata()
self.assertEqual("new_group_name", found_dict['user_group'])
def test_remove_dt_happy_flow(self):
"""
Happy flow for removing a file related to a DataType.
"""
folder_path = self.files_helper.get_project_folder(self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
open(datatype.get_storage_file_path(), 'w')
self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!")
self.files_helper.remove_datatype(datatype)
self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not deleted!")
def test_remove_dt_non_existent(self):
"""
Try to call remove on a dataType with no H5 file.
Should work.
"""
folder_path = self.files_helper.get_project_folder(self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
self.assertFalse(os.path.exists(datatype.get_storage_file_path()))
self.files_helper.remove_datatype(datatype)
def test_move_datatype(self):
"""
Make sure associated H5 file is moved to a correct new location.
"""
folder_path = self.files_helper.get_project_folder(self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
open(datatype.get_storage_file_path(), 'w')
self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!")
self.files_helper.move_datatype(datatype, self.PROJECT_NAME + '11', "43")
self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not moved!")
datatype.storage_path = self.files_helper.get_project_folder(self.PROJECT_NAME + '11', "43")
self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!")
def test_find_relative_path(self):
"""
Tests that relative path is computed properly.
"""
rel_path = self.files_helper.find_relative_path("/root/up/to/here/test/it/now", "/root/up/to/here")
self.assertEqual(rel_path, os.sep.join(["test", "it", "now"]), "Did not extract relative path as expected.")
def test_remove_files_valid(self):
"""
Pass a valid list of files and check they are all removed.
"""
file_list = ["test1", "test2", "test3"]
for file_n in file_list:
fp = open(file_n, 'w')
fp.write('test')
fp.close()
for file_n in file_list:
self.assertTrue(os.path.isfile(file_n))
self.files_helper.remove_files(file_list)
for file_n in file_list:
self.assertFalse(os.path.isfile(file_n))
def test_remove_folder(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
os.mkdir(folder_name)
self.assertTrue(os.path.isdir(folder_name), "Folder should be created.")
self.files_helper.remove_folder(folder_name)
self.assertFalse(os.path.isdir(folder_name), "Folder should be deleted.")
def test_remove_folder_non_existing_ignore_exc(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.")
self.files_helper.remove_folder(folder_name, ignore_errors=True)
def test_remove_folder_non_existing(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.")
self.assertRaises(FileStructureException, self.files_helper.remove_folder, folder_name, False)
class FlowController(BaseController):
"""
This class takes care of executing steps in projects.
"""
def __init__(self):
BaseController.__init__(self)
self.context = SelectedAdapterContext()
self.files_helper = FilesHelper()
@expose_page
@settings
@context_selected
def step_analyzers(self):
"""
Choose exact action/adapter for current step.
"""
try:
analyze_category, groups = self.flow_service.get_analyze_groups()
step_name = analyze_category.displayname.lower()
template_specification = dict(mainContent="header_menu",
section_name=step_name,
controlPage=None,
title="Select an analyzer",
displayControl=False)
adapters_list = []
for adapter_group in groups:
if len(adapter_group.children) > 1:
ids = [str(child.id) for child in adapter_group.children]
ids = ','.join(ids)
adapter_link = '/flow/show_group_of_algorithms/' + str(
analyze_category.id) + "/" + ids
else:
adapter_link = self.get_url_adapter(
analyze_category.id, adapter_group.children[0].id)
adapters_list.append({
common.KEY_TITLE:
adapter_group.name,
'link':
adapter_link,
'description':
adapter_group.description,
'subsection':
adapter_group.children[0].subsection_name
})
self.analyze_adapters = adapters_list
template_specification[common.KEY_SUBMENU_LIST] = adapters_list
return self.fill_default_attributes(template_specification)
except ValueError:
message = 'Could not load analyzers!'
common.set_warning_message(message)
self.logger.warning(message)
raise cherrypy.HTTPRedirect('/tvb')
@expose_page
@settings
@context_selected
def step_connectivity(self):
"""
Display menu for Connectivity Footer tab.
"""
template_specification = dict(mainContent="header_menu",
section_name='connectivity',
controlPage=None,
title="Select an algorithm",
displayControl=False,
subsection_name='step',
submenu_list=self.connectivity_submenu)
return self.fill_default_attributes(template_specification)
@staticmethod
def _compute_back_link(back_indicator, project):
"""
Based on a simple indicator, compute URL for anchor BACK.
"""
if back_indicator is None:
# This applies to Connectivity and other visualizers when RELAUNCH button is used from Operation page.
back_page_link = None
elif back_indicator == 'burst':
back_page_link = "/burst"
elif back_indicator == 'operations':
back_page_link = '/project/viewoperations/' + str(project.id)
else:
back_page_link = '/project/editstructure/' + str(project.id)
return back_page_link
@expose_page
@settings
@context_selected
def show_group_of_algorithms(self, step_key, algorithm_ids):
project = common.get_current_project()
category = self.flow_service.get_category_by_id(step_key)
algorithms = []
for i in algorithm_ids.split(','):
algorithm_id = int(i)
algorithm = self.flow_service.get_algorithm_by_identifier(
algorithm_id)
algorithm.link = self.get_url_adapter(step_key, algorithm_id)
algorithm.input_tree = self.flow_service.prepare_adapter(algorithm)
algorithms.append(algorithm)
template_specification = dict(
mainContent="flow/algorithms_list",
algorithms=algorithms,
title="Select an algorithm",
section_name=category.displayname.lower())
self._populate_section(algorithms[0], template_specification)
self.fill_default_attributes(template_specification,
algorithms[0].group_name)
return template_specification
@expose_page
@settings
@context_selected
def prepare_group_launch(self, group_gid, step_key, algorithm_id, **data):
"""
Receives as input a group gid and an algorithm given by category and id, along
with data that gives the name of the required input parameter for the algorithm.
Having these generate a range of GID's for all the DataTypes in the group and
launch a new operation group.
"""
prj_service = ProjectService()
dt_group = prj_service.get_datatypegroup_by_gid(group_gid)
datatypes = prj_service.get_datatypes_from_datatype_group(dt_group.id)
range_param_name = data.pop('range_param_name')
data[RANGE_PARAMETER_1] = range_param_name
data[range_param_name] = ','.join(dt.gid for dt in datatypes)
OperationService().group_operation_launch(
common.get_logged_user().id,
common.get_current_project().id, int(algorithm_id), int(step_key),
**data)
redirect_url = self._compute_back_link('operations',
common.get_current_project())
raise cherrypy.HTTPRedirect(redirect_url)
@expose_page
@settings
@context_selected
def default(self,
step_key,
adapter_key,
cancel=False,
back_page=None,
not_reset=False,
**data):
"""
Render a specific adapter.
'data' are arguments for POST
"""
project = common.get_current_project()
algorithm = self.flow_service.get_algorithm_by_identifier(adapter_key)
back_page_link = self._compute_back_link(back_page, project)
if algorithm is None:
raise cherrypy.HTTPRedirect("/tvb?error=True")
if cherrypy.request.method == 'POST' and cancel:
raise cherrypy.HTTPRedirect(back_page_link)
submit_link = self.get_url_adapter(step_key, adapter_key, back_page)
is_burst = back_page not in ['operations', 'data']
if cherrypy.request.method == 'POST':
data[common.KEY_ADAPTER] = adapter_key
template_specification = self.execute_post(project.id, submit_link,
step_key, algorithm,
**data)
self._populate_section(algorithm, template_specification, is_burst)
else:
if (('Referer' not in cherrypy.request.headers or
('Referer' in cherrypy.request.headers
and 'step' not in cherrypy.request.headers['Referer']))
and 'View' in algorithm.algorithm_category.displayname):
# Avoid reset in case of Visualizers, as a supplementary GET
not_reset = True
template_specification = self.get_template_for_adapter(
project.id,
step_key,
algorithm,
submit_link,
not not_reset,
is_burst=is_burst)
if template_specification is None:
raise cherrypy.HTTPRedirect('/tvb')
if KEY_CONTROLLS not in template_specification:
template_specification[KEY_CONTROLLS] = None
if common.KEY_SUBMIT_LINK not in template_specification:
template_specification[common.KEY_SUBMIT_LINK] = submit_link
if KEY_CONTENT not in template_specification:
template_specification[KEY_CONTENT] = "flow/full_adapter_interface"
template_specification[common.KEY_DISPLAY_MENU] = False
else:
template_specification[common.KEY_DISPLAY_MENU] = True
template_specification[common.KEY_BACK_PAGE] = back_page_link
template_specification[common.KEY_ADAPTER] = adapter_key
template_specification[ABCDisplayer.KEY_IS_ADAPTER] = True
self.fill_default_attributes(template_specification,
algorithm.displayname)
return template_specification
@expose_fragment("flow/reduce_dimension_select")
def gettemplatefordimensionselect(self,
entity_gid=None,
select_name="",
reset_session='False',
parameters_prefix="dimensions",
required_dimension=1,
expected_shape="",
operations=""):
"""
Returns the HTML which contains the selects components which allows the user
to reduce the dimension of a multi-dimensional array.
We try to obtain the aggregation_functions from the entity, which is a list of lists.
For each dimension should be a list with the supported aggregation functions. We
create a DICT for each of those lists. The key will be the name of the function and
the value will be its label.
entity_gid
the GID of the entity for which is displayed the component
select_name
the name of the parent select. The select in which
is displayed the entity with the given GID
parameters_prefix
a string which will be used for computing the names of the component
required_dimension
the expected dimension for the resulted array
expected_shape and operations
used for applying conditions on the resulted array
e.g.: If the resulted array is a 3D array and we want that the length of the second
dimension to be smaller then 512 then the expected_shape and operations should be:
``expected_shape=x,512,x`` and ``operations='x,<,x``
"""
template_params = {
"select_name": "",
"data": [],
"parameters_prefix": parameters_prefix,
"array_shape": "",
"required_dimension": required_dimension,
"currentDim": "",
"required_dim_msg": "",
"expected_shape": expected_shape,
"operations": operations
}
# if reload => populate the selected values
session_dict = self.context.get_current_default()
dimensions = {1: [0], 3: [0]}
selected_agg_functions = {}
if not string2bool(str(reset_session)) and session_dict is not None:
starts_with_str = select_name + "_" + parameters_prefix + "_"
ui_sel_items = dict((k, v) for k, v in session_dict.items()
if k.startswith(starts_with_str))
from tvb.datatypes.arrays import MappedArray
dimensions, selected_agg_functions, required_dimension, _ = MappedArray(
).parse_selected_items(ui_sel_items)
template_params["selected_items"] = dimensions
template_params["selected_functions"] = selected_agg_functions
aggregation_functions = []
default_agg_functions = self.accepted__aggregation_functions()
labels_set = ["Time", "Channel", "Line"]
if entity_gid is not None:
actual_entity = ABCAdapter.load_entity_by_gid(entity_gid)
if hasattr(actual_entity, 'shape'):
array_shape = actual_entity.shape
new_shape, current_dim = self._compute_current_dimension(
list(array_shape), dimensions, selected_agg_functions)
if required_dimension is not None and current_dim != int(
required_dimension):
template_params[
"required_dim_msg"] = "Please select a " + str(
required_dimension) + "D array"
if not current_dim:
template_params["currentDim"] = "1 element"
else:
template_params["currentDim"] = str(
current_dim) + "D array"
template_params["array_shape"] = json.dumps(new_shape)
if hasattr(actual_entity, 'dimensions_labels'
) and actual_entity.dimensions_labels is not None:
labels_set = actual_entity.dimensions_labels
# make sure there exists labels for each dimension
while len(labels_set) < len(array_shape):
labels_set.append("Undefined")
if (hasattr(actual_entity, 'aggregation_functions')
and actual_entity.aggregation_functions is not None
and len(actual_entity.aggregation_functions)
== len(array_shape)):
# will be a list of lists of aggregation functions
defined_functions = actual_entity.aggregation_functions
for function in defined_functions:
if not len(function):
aggregation_functions.append({})
else:
func_dict = {}
for function_key in function:
func_dict[
function_key] = default_agg_functions[
function_key]
aggregation_functions.append(func_dict)
else:
for _ in array_shape:
aggregation_functions.append(default_agg_functions)
result = []
for i, shape in enumerate(array_shape):
labels = []
values = []
for j in range(shape):
labels.append(labels_set[i] + " " + str(j))
values.append(entity_gid + "_" + str(i) + "_" + str(j))
result.append([labels, values, aggregation_functions[i]])
template_params["select_name"] = select_name
template_params["data"] = result
return template_params
return template_params
@staticmethod
def _compute_current_dimension(array_shape, selected_items,
selected_functions):
"""
If the user reloads an operation we have to compute the current dimension of the array
and also the shape of the array based on his selections
"""
current_dim = len(array_shape)
for i in range(len(array_shape)):
if i in selected_items and len(selected_items[i]) > 0:
array_shape[i] = len(selected_items[i])
if len(selected_items[i]) == 1:
current_dim -= 1
if i in selected_functions and selected_functions[i] != 'none':
array_shape[i] = 1
if i not in selected_items or len(selected_items[i]) > 1:
current_dim -= 1
return array_shape, current_dim
@staticmethod
def accepted__aggregation_functions():
"""
Returns the list of aggregation functions that may be
applied on arrays.
"""
return {"sum": "Sum", "average": "Average"}
@expose_fragment("flow/type2component/datatype2select_simple")
def getfiltereddatatypes(self, name, parent_div, tree_session_key,
filters):
"""
Given the name from the input tree, the dataType required and a number of
filters, return the available dataType that satisfy the conditions imposed.
"""
previous_tree = self.context.get_session_tree_for_key(tree_session_key)
if previous_tree is None:
common.set_error_message(
"Adapter Interface not in session for filtering!")
raise cherrypy.HTTPRedirect("/tvb?error=True")
current_node = self._get_node(previous_tree, name)
if current_node is None:
raise Exception("Could not find node :" + name)
datatype = current_node[ABCAdapter.KEY_DATATYPE]
filters = json.loads(filters)
availablefilter = json.loads(
FilterChain.get_filters_for_type(datatype))
for i, filter_ in enumerate(filters[FILTER_FIELDS]):
# Check for filter input of type 'date' as these need to be converted
if filter_ in availablefilter and availablefilter[filter_][
FILTER_TYPE] == 'date':
try:
temp_date = string2date(filters[FILTER_VALUES][i], False)
filters[FILTER_VALUES][i] = temp_date
except ValueError:
raise
# In order for the filter object not to "stack up" on multiple calls to
# this method, create a deepCopy to work with
if ABCAdapter.KEY_CONDITION in current_node:
new_filter = copy.deepcopy(current_node[ABCAdapter.KEY_CONDITION])
else:
new_filter = FilterChain()
new_filter.fields.extend(filters[FILTER_FIELDS])
new_filter.operations.extend(filters[FILTER_OPERATIONS])
new_filter.values.extend(filters[FILTER_VALUES])
# Get dataTypes that match the filters from DB then populate with values
values, total_count = InputTreeManager(
).populate_option_values_for_dtype(common.get_current_project().id,
datatype, new_filter,
self.context.get_current_step())
# Create a dictionary that matches what the template expects
parameters = {
ABCAdapter.KEY_NAME: name,
ABCAdapter.KEY_FILTERABLE: availablefilter,
ABCAdapter.KEY_TYPE: ABCAdapter.TYPE_SELECT,
ABCAdapter.KEY_OPTIONS: values,
ABCAdapter.KEY_DATATYPE: datatype
}
if total_count > MAXIMUM_DATA_TYPES_DISPLAYED:
parameters[KEY_WARNING] = WARNING_OVERFLOW
if ABCAdapter.KEY_REQUIRED in current_node:
parameters[ABCAdapter.KEY_REQUIRED] = current_node[
ABCAdapter.KEY_REQUIRED]
if len(values) > 0 and string2bool(
str(parameters[ABCAdapter.KEY_REQUIRED])):
parameters[ABCAdapter.KEY_DEFAULT] = str(
values[-1][ABCAdapter.KEY_VALUE])
previous_selected = self.context.get_current_default(name)
if previous_selected in [str(vv['value']) for vv in values]:
parameters[ABCAdapter.KEY_DEFAULT] = previous_selected
template_specification = {
"inputRow": parameters,
"disabled": False,
"parentDivId": parent_div,
common.KEY_SESSION_TREE: tree_session_key
}
return self.fill_default_attributes(template_specification)
def _get_node(self, input_tree, name):
"""
Given a input tree and a variable name, check to see if any default filters exist.
"""
for entry in input_tree:
if (ABCAdapter.KEY_DATATYPE in entry
and ABCAdapter.KEY_NAME in entry
and str(entry[ABCAdapter.KEY_NAME]) == str(name)):
return entry
if entry.get(ABCAdapter.KEY_ATTRIBUTES) is not None:
in_attr = self._get_node(entry[ABCAdapter.KEY_ATTRIBUTES],
name)
if in_attr is not None:
return in_attr
if entry.get(ABCAdapter.KEY_OPTIONS) is not None:
in_options = self._get_node(entry[ABCAdapter.KEY_OPTIONS],
name)
if in_options is not None:
return in_options
return None
def execute_post(self, project_id, submit_url, step_key, algorithm,
**data):
""" Execute HTTP POST on a generic step."""
errors = None
adapter_instance = ABCAdapter.build_adapter(algorithm)
try:
form = adapter_instance.get_form()(project_id=project_id)
form.fill_from_post(data)
dt_dict = None
if form.validate():
dt_dict = form.get_dict()
if dt_dict is None:
raise formencode.Invalid(
"Could not build a dict out of this form!", {},
None,
error_dict=form.get_errors_dict())
adapter_instance.submit_form(form)
result = self.flow_service.fire_operation(adapter_instance,
common.get_logged_user(),
project_id, **dt_dict)
# Store input data in session, for informing user of it.
step = self.flow_service.get_category_by_id(step_key)
if not step.rawinput:
self.context.add_adapter_to_session(None, None,
copy.deepcopy(data))
if isinstance(adapter_instance, ABCDisplayer):
if isinstance(result, dict):
result[common.
KEY_OPERATION_ID] = adapter_instance.operation_id
return result
else:
common.set_error_message(
"Invalid result returned from Displayer! Dictionary is expected!"
)
else:
if isinstance(result, list):
result = "Launched %s operations." % len(result)
common.set_important_message(str(result))
except formencode.Invalid as excep:
errors = excep.unpack_errors()
common.set_error_message("Invalid form inputs")
self.logger.warning("Invalid form inputs %s" % errors)
except OperationException as excep1:
self.logger.exception("Error while executing a Launch procedure:" +
excep1.message)
common.set_error_message(excep1.message)
previous_step = self.context.get_current_substep()
should_reset = previous_step is None or data.get(
common.KEY_ADAPTER) != previous_step
template_specification = self.get_template_for_adapter(
project_id, step_key, algorithm, submit_url, should_reset)
if (errors is not None) and (template_specification is not None):
template_specification[common.KEY_ERRORS] = errors
template_specification[
common.KEY_OPERATION_ID] = adapter_instance.operation_id
return template_specification
def get_template_for_adapter(self,
project_id,
step_key,
stored_adapter,
submit_url,
session_reset=True,
is_burst=True):
""" Get Input HTML Interface template or a given adapter """
try:
if session_reset:
self.context.clean_from_session()
group = None
# Cache some values in session, for performance
previous_tree = self.context.get_current_input_tree()
previous_sub_step = self.context.get_current_substep()
category = self.flow_service.get_category_by_id(step_key)
title = "Fill parameters for step " + category.displayname.lower()
if group:
title = title + " - " + group.displayname
adapter_instance = self.flow_service.prepare_adapter(
stored_adapter)
if adapter_instance.get_input_tree() is None:
adapter_form = self.flow_service.prepare_adapter_form(
adapter_instance, project_id)
template_specification = dict(submitLink=submit_url,
form=adapter_form,
title=title)
# TODO: to be removed when all forms are migrated
else:
if not session_reset and previous_tree is not None and previous_sub_step == stored_adapter.id:
adapter_interface = previous_tree
else:
adapter_interface = self.flow_service.prepare_adapter_tree_interface(
adapter_instance, project_id,
stored_adapter.fk_category)
self.context.add_adapter_to_session(
stored_adapter, adapter_interface)
current_defaults = self.context.get_current_default()
if current_defaults is not None:
# Change default values in tree, according to selected input
adapter_interface = InputTreeManager.fill_defaults(
adapter_interface, current_defaults)
template_specification = dict(submitLink=submit_url,
inputList=adapter_interface,
title=title)
self._populate_section(stored_adapter, template_specification,
is_burst)
return template_specification
except OperationException as oexc:
self.logger.error("Inconsistent Adapter")
self.logger.exception(oexc)
common.set_warning_message(
'Inconsistent Adapter! Please review the link (development problem)!'
)
return None
@cherrypy.expose
@handle_error(redirect=False)
@check_user
def readserverstaticfile(self, coded_path):
"""
Retrieve file from Local storage, having a File System Path.
"""
try:
with open(url2path(coded_path), "rb") as f:
return f.read()
except Exception as excep:
self.logger.error("Could not retrieve file from path:" +
str(coded_path))
self.logger.exception(excep)
def _read_datatype_attribute(self,
entity_gid,
dataset_name,
datatype_kwargs='null',
**kwargs):
self.logger.debug("Starting to read HDF5: " + entity_gid + "/" +
dataset_name + "/" + str(kwargs))
entity = ABCAdapter.load_entity_by_gid(entity_gid)
entity_dt = h5.load_from_index(entity)
datatype_kwargs = json.loads(datatype_kwargs)
if datatype_kwargs:
for key, value in six.iteritems(datatype_kwargs):
kwargs[key] = ABCAdapter.load_entity_by_gid(value)
result = getattr(entity_dt, dataset_name)
if callable(result):
if kwargs:
result = result(**kwargs)
else:
result = result()
return result
@expose_json
def invoke_adapter(self, algo_id, method_name, entity_gid, **kwargs):
algorithm = self.flow_service.get_algorithm_by_identifier(algo_id)
adapter_instance = ABCAdapter.build_adapter(algorithm)
entity = ABCAdapter.load_entity_by_gid(entity_gid)
storage_path = self.files_helper.get_project_folder(
entity.parent_operation.project, str(entity.fk_from_operation))
adapter_instance.storage_path = storage_path
method = getattr(adapter_instance, method_name)
if kwargs:
return method(entity_gid, **kwargs)
return method(entity_gid)
@expose_json
def read_from_h5_file(self,
entity_gid,
method_name,
flatten=False,
datatype_kwargs='null',
**kwargs):
self.logger.debug("Starting to read HDF5: " + entity_gid + "/" +
method_name + "/" + str(kwargs))
entity = ABCAdapter.load_entity_by_gid(entity_gid)
entity_h5 = h5.h5_file_for_index(entity)
datatype_kwargs = json.loads(datatype_kwargs)
if datatype_kwargs:
for key, value in six.iteritems(datatype_kwargs):
kwargs[key] = ABCAdapter.load_entity_by_gid(value)
result = getattr(entity_h5, method_name)
if kwargs:
result = result(**kwargs)
else:
result = result()
entity_h5.close()
return self._prepare_result(result, flatten)
@expose_json
def read_datatype_attribute(self,
entity_gid,
dataset_name,
flatten=False,
datatype_kwargs='null',
**kwargs):
"""
Retrieve from a given DataType a property or a method result.
:returns: JSON representation of the attribute.
:param entity_gid: GID for DataType entity
:param dataset_name: name of the dataType property /method
:param flatten: result should be flatten before return (use with WebGL data mainly e.g vertices/triangles)
Ignored if the attribute is not an ndarray
:param datatype_kwargs: if passed, will contain a dictionary of type {'name' : 'gid'}, and for each such
pair, a load_entity will be performed and kwargs will be updated to contain the result
:param kwargs: extra parameters to be passed when dataset_name is method.
"""
result = self._read_datatype_attribute(entity_gid, dataset_name,
datatype_kwargs, **kwargs)
return self._prepare_result(result, flatten)
def _prepare_result(self, result, flatten):
if isinstance(result, numpy.ndarray):
# for ndarrays honor the flatten kwarg and convert to lists as ndarrs are not json-able
if flatten is True or flatten == "True":
result = result.flatten()
return result.tolist()
else:
return result
@expose_numpy_array
def read_binary_datatype_attribute(self,
entity_gid,
dataset_name,
datatype_kwargs='null',
**kwargs):
return self._read_datatype_attribute(entity_gid, dataset_name,
datatype_kwargs, **kwargs)
@expose_fragment("flow/genericAdapterFormFields")
def get_simple_adapter_interface(self,
algorithm_id,
parent_div='',
is_uploader=False):
"""
AJAX exposed method. Will return only the interface for a adapter, to
be used when tabs are needed.
"""
curent_project = common.get_current_project()
is_uploader = string2bool(is_uploader)
template_specification = self.get_adapter_template(
curent_project.id, algorithm_id, is_uploader)
template_specification[common.KEY_PARENT_DIV] = parent_div
return self.fill_default_attributes(template_specification)
@expose_fragment("flow/full_adapter_interface")
def getadapterinterface(self, project_id, algorithm_id, back_page=None):
"""
AJAX exposed method. Will return only a piece of a page,
to be integrated as part in another page.
"""
template_specification = self.get_adapter_template(
project_id, algorithm_id, False, back_page)
template_specification["isCallout"] = True
return self.fill_default_attributes(template_specification)
def get_adapter_template(self,
project_id,
algorithm_id,
is_upload=False,
back_page=None):
"""
Get the template for an adapter based on the algo group id.
"""
if not (project_id and int(project_id) and
(algorithm_id is not None) and int(algorithm_id)):
return ""
algorithm = self.flow_service.get_algorithm_by_identifier(algorithm_id)
if is_upload:
submit_link = "/project/launchloader/" + str(
project_id) + "/" + str(algorithm_id)
else:
submit_link = self.get_url_adapter(algorithm.fk_category,
algorithm.id, back_page)
current_step = self.context.get_current_substep()
if current_step is None or str(current_step) != str(algorithm_id):
self.context.clean_from_session()
template_specification = self.get_template_for_adapter(
project_id, algorithm.fk_category, algorithm, submit_link,
is_upload)
if template_specification is None:
return ""
template_specification[common.KEY_DISPLAY_MENU] = not is_upload
return template_specification
@cherrypy.expose
@handle_error(redirect=True)
@context_selected
def reloadoperation(self, operation_id, **_):
"""Redirect to Operation Input selection page,
with input data already selected."""
operation = self.flow_service.load_operation(operation_id)
data = parse_json_parameters(operation.parameters)
self.context.add_adapter_to_session(operation.algorithm, None, data)
category_id = operation.algorithm.fk_category
algo_id = operation.fk_from_algo
raise cherrypy.HTTPRedirect("/flow/" + str(category_id) + "/" +
str(algo_id) + "?not_reset=True")
@cherrypy.expose
@handle_error(redirect=True)
@context_selected
def reload_burst_operation(self, operation_id, is_group, **_):
"""
Find out from which burst was this operation launched. Set that burst as the selected one and
redirect to the burst page.
"""
is_group = int(is_group)
if not is_group:
operation = self.flow_service.load_operation(int(operation_id))
else:
op_group = ProjectService.get_operation_group_by_id(operation_id)
first_op = ProjectService.get_operations_in_group(op_group)[0]
operation = self.flow_service.load_operation(int(first_op.id))
operation.burst.prepare_after_load()
common.add2session(common.KEY_BURST_CONFIG, operation.burst)
raise cherrypy.HTTPRedirect("/burst/")
@expose_json
def stop_operation(self, operation_id, is_group, remove_after_stop=False):
"""
Stop the operation given by operation_id. If is_group is true stop all the
operations from that group.
"""
operation_service = OperationService()
result = False
if int(is_group) == 0:
result = operation_service.stop_operation(operation_id)
if remove_after_stop:
ProjectService().remove_operation(operation_id)
else:
op_group = ProjectService.get_operation_group_by_id(operation_id)
operations_in_group = ProjectService.get_operations_in_group(
op_group)
for operation in operations_in_group:
tmp_res = operation_service.stop_operation(operation.id)
if remove_after_stop:
ProjectService().remove_operation(operation.id)
result = result or tmp_res
return result
@expose_json
def stop_burst_operation(self,
operation_id,
is_group,
remove_after_stop=False):
"""
For a given operation id that is part of a burst just stop the given burst.
:returns True when stopped operation was successfully.
"""
operation_id = int(operation_id)
if int(is_group) == 0:
operation = self.flow_service.load_operation(operation_id)
else:
op_group = ProjectService.get_operation_group_by_id(operation_id)
first_op = ProjectService.get_operations_in_group(op_group)[0]
operation = self.flow_service.load_operation(int(first_op.id))
try:
burst_service = BurstService()
result = burst_service.stop_burst(operation.burst)
if remove_after_stop:
current_burst = common.get_from_session(
common.KEY_BURST_CONFIG)
if current_burst and current_burst.id == operation.burst.id:
common.remove_from_session(common.KEY_BURST_CONFIG)
result = burst_service.cancel_or_remove_burst(
operation.burst.id) or result
return result
except Exception as ex:
self.logger.exception(ex)
return False
def fill_default_attributes(self, template_dictionary, title='-'):
"""
Overwrite base controller to add required parameters for adapter templates.
"""
if common.KEY_TITLE not in template_dictionary:
template_dictionary[common.KEY_TITLE] = title
if common.KEY_PARENT_DIV not in template_dictionary:
template_dictionary[common.KEY_PARENT_DIV] = ''
if common.KEY_PARAMETERS_CONFIG not in template_dictionary:
template_dictionary[common.KEY_PARAMETERS_CONFIG] = False
template_dictionary[
common.KEY_INCLUDE_RESOURCES] = 'flow/included_resources'
BaseController.fill_default_attributes(self, template_dictionary)
return template_dictionary
##### Below this point are operations that might be moved to different #####
##### controller #####
NEW_SELECTION_NAME = 'New selection'
def _get_available_selections(self, datatype_gid):
"""
selection retrieval common to selection component and connectivity selection
"""
curent_project = common.get_current_project()
selections = self.flow_service.get_selections_for_project(
curent_project.id, datatype_gid)
names, sel_values = [], []
for selection in selections:
names.append(selection.ui_name)
sel_values.append(selection.selected_nodes)
return names, sel_values
@expose_fragment('visualizers/commons/channel_selector_opts')
def get_available_selections(self, **data):
sel_names, sel_values = self._get_available_selections(
data['datatype_gid'])
return dict(namedSelections=list(zip(sel_names, sel_values)))
@expose_json
def store_measure_points_selection(self, ui_name, **data):
"""
Save a MeasurePoints selection (new or update existing entity).
"""
if ui_name and ui_name != self.NEW_SELECTION_NAME:
sel_project_id = common.get_current_project().id
# client sends integers as strings:
selection = json.dumps(
[int(s) for s in json.loads(data['selection'])])
datatype_gid = data['datatype_gid']
self.flow_service.save_measure_points_selection(
ui_name, selection, datatype_gid, sel_project_id)
return [True, 'Selection saved successfully.']
else:
error_msg = self.NEW_SELECTION_NAME + " or empty name are not valid as selection names."
return [False, error_msg]
@expose_fragment(
"visualizers/pse_discrete/inserting_new_threshold_spec_bar")
def create_row_of_specs(self, count):
return dict(id_increment_count=count)
@expose_json
def store_pse_filter(self, config_name, **data):
# this will need to be updated in such a way that the expose_json actually gets used
## also this is going to be changed to be storing through the flow service and dao. Stay updated
try:
##this is to check whether there is already an entry in the
for i, (name, Val) in enumerate(self.PSE_names_list):
if name == config_name:
self.PSE_names_list[i] = (
config_name,
(data['threshold_value'] + "," +
data['threshold_type'] + "," + data['not_presence'])
) # replace the previous occurence of the config name, and carry on.
self.get_pse_filters()
return [
True, 'Selected Text stored, and selection updated'
]
self.PSE_names_list.append(
(config_name,
(data['threshold_value'] + "," + data['threshold_type'] +
"," + data['not_presence'])))
except AttributeError:
self.PSE_names_list = [
(config_name,
(data['threshold_value'] + "," + data['threshold_type'] +
"," + data['not_presence']))
]
self.get_pse_filters()
return [True, 'Selected Text stored, and selection updated']
@expose_fragment("visualizers/commons/channel_selector_opts")
def get_pse_filters(self):
try:
return dict(namedSelections=self.PSE_names_list)
except AttributeError:
return dict(
namedSelections=[]
) # this will give us back atleast the New Selection option in the select
except:
raise
@expose_json
def store_exploration_section(self, val_range, step, dt_group_guid):
"""
Launching method for further simulations.
"""
range_list = [float(num) for num in val_range.split(",")]
step_list = [float(num) for num in step.split(",")]
datatype_group_ob = ProjectService().get_datatypegroup_by_gid(
dt_group_guid)
operation_grp = datatype_group_ob.parent_operation_group
operation_obj = self.flow_service.load_operation(
datatype_group_ob.fk_from_operation)
parameters = json.loads(operation_obj.parameters)
range1name, range1_dict = json.loads(operation_grp.range1)
range2name, range2_dict = json.loads(operation_grp.range2)
parameters[RANGE_PARAMETER_1] = range1name
parameters[RANGE_PARAMETER_2] = range2name
##change the existing simulator parameters to be min max step types
range1_dict = {
constants.ATT_MINVALUE: range_list[0],
constants.ATT_MAXVALUE: range_list[1],
constants.ATT_STEP: step_list[0]
}
range2_dict = {
constants.ATT_MINVALUE: range_list[2],
constants.ATT_MAXVALUE: range_list[3],
constants.ATT_STEP: step_list[1]
}
parameters[range1name] = json.dumps(
range1_dict) # this is for the x axis parameter
parameters[range2name] = json.dumps(
range2_dict) # this is for the y axis parameter
OperationService().group_operation_launch(
common.get_logged_user().id,
common.get_current_project().id, operation_obj.algorithm.id,
operation_obj.algorithm.fk_category, datatype_group_ob,
**parameters)
return [True, 'Stored the exploration material successfully']
class FlowService:
"""
Service Layer for all TVB generic Work-Flow operations.
"""
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.file_helper = FilesHelper()
self.input_tree_manager = InputTreeManager()
def get_category_by_id(self, identifier):
""" Pass to DAO the retrieve of category by ID operation."""
return dao.get_category_by_id(identifier)
@staticmethod
def get_raw_categories():
""":returns: AlgorithmCategory list of entities that have results in RAW state (Creators/Uploaders)"""
return dao.get_raw_categories()
@staticmethod
def get_visualisers_category():
"""Retrieve all Algorithm categories, with display capability"""
result = dao.get_visualisers_categories()
if not result:
raise ValueError("View Category not found!!!")
return result[0]
@staticmethod
def get_algorithm_by_identifier(ident):
"""
Retrieve Algorithm entity by ID.
Return None, if ID is not found in DB.
"""
return dao.get_algorithm_by_id(ident)
@staticmethod
def load_operation(operation_id):
""" Retrieve previously stored Operation from DB, and load operation.burst attribute"""
operation = dao.get_operation_by_id(operation_id)
operation.burst = dao.get_burst_for_operation_id(operation_id)
return operation
@staticmethod
def get_operation_numbers(proj_id):
""" Count total number of operations started for current project. """
return dao.get_operation_numbers(proj_id)
def prepare_adapter(self, project_id, stored_adapter):
"""
Having a StoredAdapter, return the Tree Adapter Interface object, populated with datatypes from 'project_id'.
"""
adapter_module = stored_adapter.module
adapter_name = stored_adapter.classname
try:
# Prepare Adapter Interface, by populating with existent data,
# in case of a parameter of type DataType.
adapter_instance = ABCAdapter.build_adapter(stored_adapter)
interface = adapter_instance.get_input_tree()
interface = self.input_tree_manager.fill_input_tree_with_options(interface, project_id, stored_adapter.fk_category)
interface = self.input_tree_manager.prepare_param_names(interface)
return interface
except Exception:
self.logger.exception('Not found:' + adapter_name + ' in:' + adapter_module)
raise OperationException("Could not prepare " + adapter_name)
@staticmethod
def get_algorithm_by_module_and_class(module, classname):
"""
Get the db entry from the algorithm table for the given module and
class.
"""
return dao.get_algorithm_by_module(module, classname)
@staticmethod
def get_available_datatypes(project_id, data_type_cls, filters=None):
"""
Return all dataTypes that match a given name and some filters.
:param data_type_cls: either a fully qualified class name or a class object
"""
return get_filtered_datatypes(project_id, data_type_cls, filters)
@staticmethod
def create_link(data_ids, project_id):
"""
For a list of dataType IDs and a project id create all the required links.
"""
for data in data_ids:
link = model.Links(data, project_id)
dao.store_entity(link)
@staticmethod
def remove_link(dt_id, project_id):
"""
Remove the link from the datatype given by dt_id to project given by project_id.
"""
link = dao.get_link(dt_id, project_id)
if link is not None:
dao.remove_entity(model.Links, link.id)
def fire_operation(self, adapter_instance, current_user, project_id, visible=True, **data):
"""
Launch an operation, specified by AdapterInstance, for CurrentUser,
Current Project and a given set of UI Input Data.
"""
operation_name = str(adapter_instance.__class__.__name__)
try:
self.logger.info("Starting operation " + operation_name)
project = dao.get_project_by_id(project_id)
tmp_folder = self.file_helper.get_project_folder(project, self.file_helper.TEMP_FOLDER)
result = OperationService().initiate_operation(current_user, project.id, adapter_instance,
tmp_folder, visible, **data)
self.logger.info("Finished operation:" + operation_name)
return result
except TVBException, excep:
self.logger.exception("Could not launch operation " + operation_name +
" with the given set of input data, because: " + excep.message)
raise OperationException(excep.message, excep)
except Exception, excep:
self.logger.exception("Could not launch operation " + operation_name + " with the given set of input data!")
raise OperationException(str(excep))
class TestSimulationResource(TransactionalTestCase):
def transactional_setup_method(self):
self.test_user = TestFactory.create_user('Rest_User')
self.test_project = TestFactory.create_project(self.test_user,
'Rest_Project')
self.simulation_resource = FireSimulationResource()
self.files_helper = FilesHelper()
def test_server_fire_simulation_inexistent_gid(self, mocker):
project_gid = "inexistent-gid"
dummy_file = FileStorage(BytesIO(b"test"), 'test.zip')
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request')
request_mock.files = {'file': dummy_file}
with pytest.raises(InvalidIdentifierException):
self.simulation_resource.post(project_gid)
def test_server_fire_simulation_no_file(self, mocker):
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request')
request_mock.files = {}
with pytest.raises(BadRequestException):
self.simulation_resource.post('')
def test_server_fire_simulation_bad_extension(self, mocker):
dummy_file = FileStorage(BytesIO(b"test"), 'test.txt')
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request')
request_mock.files = {'file': dummy_file}
with pytest.raises(BadRequestException):
self.simulation_resource.post('')
def test_server_fire_simulation(self, mocker, connectivity_factory):
input_folder = self.files_helper.get_project_folder(self.test_project)
sim_dir = os.path.join(input_folder, 'test_sim')
if not os.path.isdir(sim_dir):
os.makedirs(sim_dir)
simulator = Simulator()
simulator.connectivity = connectivity_factory()
sim_serializer = SimulatorSerializer()
sim_serializer.serialize_simulator(simulator, simulator.gid.hex, None,
sim_dir)
zip_filename = shutil.make_archive(sim_dir, 'zip', input_folder)
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request')
fp = open(zip_filename, 'rb')
request_mock.files = {
'file': FileStorage(fp, os.path.basename(zip_filename))
}
def launch_sim(self, user_id, project, algorithm, zip_folder_path,
simulator_file):
return Operation('', '', '', {})
# Mock simulation launch
mocker.patch.object(SimulatorService, 'prepare_simulation_on_server',
launch_sim)
operation_gid, status = self.simulation_resource.post(
self.test_project.gid)
fp.close()
assert type(operation_gid) is str
assert status == 201
def transactional_teardown_method(self):
self.files_helper.remove_project_structure(self.test_project.name)
class TestProjectService(TransactionalTestCase):
"""
This class contains tests for the tvb.core.services.project_service module.
"""
def transactional_setup_method(self):
"""
Reset the database before each test.
"""
self.project_service = ProjectService()
self.structure_helper = FilesHelper()
self.test_user = TestFactory.create_user()
def transactional_teardown_method(self):
"""
Remove project folders and clean up database.
"""
created_projects = dao.get_projects_for_user(self.test_user.id)
for project in created_projects:
self.structure_helper.remove_project_structure(project.name)
self.delete_project_folders()
def test_create_project_happy_flow(self):
user1 = TestFactory.create_user('test_user1')
user2 = TestFactory.create_user('test_user2')
initial_projects = dao.get_projects_for_user(self.test_user.id)
assert len(initial_projects) == 0, "Database reset probably failed!"
TestFactory.create_project(self.test_user,
'test_project',
"description",
users=[user1.id, user2.id])
resulting_projects = dao.get_projects_for_user(self.test_user.id)
assert len(
resulting_projects) == 1, "Project with valid data not inserted!"
project = resulting_projects[0]
assert project.name == "test_project", "Invalid retrieved project name"
assert project.description == "description", "Description do no match"
users_for_project = dao.get_members_of_project(project.id)
for user in users_for_project:
assert user.id in [user1.id, user2.id,
self.test_user.id], "Users not stored properly."
assert os.path.exists(
os.path.join(TvbProfile.current.TVB_STORAGE,
FilesHelper.PROJECTS_FOLDER,
"test_project")), "Folder for project was not created"
def test_create_project_empty_name(self):
"""
Creating a project with an empty name.
"""
data = dict(name="", description="test_description", users=[])
initial_projects = dao.get_projects_for_user(self.test_user.id)
assert len(initial_projects) == 0, "Database reset probably failed!"
with pytest.raises(ProjectServiceException):
self.project_service.store_project(self.test_user, True, None,
**data)
def test_edit_project_happy_flow(self):
"""
Standard flow for editing an existing project.
"""
selected_project = TestFactory.create_project(self.test_user,
'test_proj')
proj_root = self.structure_helper.get_project_folder(selected_project)
initial_projects = dao.get_projects_for_user(self.test_user.id)
assert len(
initial_projects) == 1, "Database initialization probably failed!"
edited_data = dict(name="test_project",
description="test_description",
users=[])
edited_project = self.project_service.store_project(
self.test_user, False, selected_project.id, **edited_data)
assert not os.path.exists(proj_root), "Previous folder not deleted"
proj_root = self.structure_helper.get_project_folder(edited_project)
assert os.path.exists(proj_root), "New folder not created!"
assert selected_project.name != edited_project.name, "Project was no changed!"
def test_edit_project_unexisting(self):
"""
Trying to edit an un-existing project.
"""
selected_project = TestFactory.create_project(self.test_user,
'test_proj')
self.structure_helper.get_project_folder(selected_project)
initial_projects = dao.get_projects_for_user(self.test_user.id)
assert len(
initial_projects) == 1, "Database initialization probably failed!"
data = dict(name="test_project",
description="test_description",
users=[])
with pytest.raises(ProjectServiceException):
self.project_service.store_project(self.test_user, False, 99,
**data)
def test_find_project_happy_flow(self):
"""
Standard flow for finding a project by it's id.
"""
initial_projects = dao.get_projects_for_user(self.test_user.id)
assert len(initial_projects) == 0, "Database reset probably failed!"
inserted_project = TestFactory.create_project(self.test_user,
'test_project')
assert self.project_service.find_project(
inserted_project.id) is not None, "Project not found !"
dao_returned_project = dao.get_project_by_id(inserted_project.id)
service_returned_project = self.project_service.find_project(
inserted_project.id)
assert dao_returned_project.id == service_returned_project.id, \
"Data returned from service is different from data returned by DAO."
assert dao_returned_project.name == service_returned_project.name, \
"Data returned from service is different than data returned by DAO."
assert dao_returned_project.description == service_returned_project.description, \
"Data returned from service is different from data returned by DAO."
assert dao_returned_project.members == service_returned_project.members, \
"Data returned from service is different from data returned by DAO."
def test_find_project_unexisting(self):
"""
Searching for an un-existing project.
"""
data = dict(name="test_project",
description="test_description",
users=[])
initial_projects = dao.get_projects_for_user(self.test_user.id)
assert len(initial_projects) == 0, "Database reset probably failed!"
project = self.project_service.store_project(self.test_user, True,
None, **data)
# fetch a likely non-existing project. Previous project id plus a 'big' offset
with pytest.raises(ProjectServiceException):
self.project_service.find_project(project.id + 1033)
def test_retrieve_projects_for_user(self):
"""
Test for retrieving the projects for a given user. One page only.
"""
initial_projects = self.project_service.retrieve_projects_for_user(
self.test_user.id)[0]
assert len(initial_projects) == 0, "Database was not reset properly!"
TestFactory.create_project(self.test_user, 'test_proj')
TestFactory.create_project(self.test_user, 'test_proj1')
TestFactory.create_project(self.test_user, 'test_proj2')
user1 = TestFactory.create_user('another_user')
TestFactory.create_project(user1, 'test_proj3')
projects = self.project_service.retrieve_projects_for_user(
self.test_user.id)[0]
assert len(projects) == 3, "Projects not retrieved properly!"
for project in projects:
assert project.name != "test_project3", "This project should not have been retrieved"
def test_retrieve_1project_3usr(self):
"""
One user as admin, two users as members, getting projects for admin and for any of
the members should return one.
"""
member1 = TestFactory.create_user("member1")
member2 = TestFactory.create_user("member2")
TestFactory.create_project(self.test_user,
'Testproject',
users=[member1.id, member2.id])
projects = self.project_service.retrieve_projects_for_user(
self.test_user.id, 1)[0]
assert len(projects) == 1, "Projects not retrieved properly!"
projects = self.project_service.retrieve_projects_for_user(
member1.id, 1)[0]
assert len(projects) == 1, "Projects not retrieved properly!"
projects = self.project_service.retrieve_projects_for_user(
member2.id, 1)[0]
assert len(projects) == 1, "Projects not retrieved properly!"
def test_retrieve_3projects_3usr(self):
"""
Three users, 3 projects. Structure of db:
proj1: {admin: user1, members: [user2, user3]}
proj2: {admin: user2, members: [user1]}
proj3: {admin: user3, members: [user1, user2]}
Check valid project returns for all the users.
"""
member1 = TestFactory.create_user("member1")
member2 = TestFactory.create_user("member2")
member3 = TestFactory.create_user("member3")
TestFactory.create_project(member1,
'TestProject1',
users=[member2.id, member3.id])
TestFactory.create_project(member2, 'TestProject2', users=[member1.id])
TestFactory.create_project(member3,
'TestProject3',
users=[member1.id, member2.id])
projects = self.project_service.retrieve_projects_for_user(
member1.id, 1)[0]
assert len(projects) == 3, "Projects not retrieved properly!"
projects = self.project_service.retrieve_projects_for_user(
member2.id, 1)[0]
assert len(projects) == 3, "Projects not retrieved properly!"
projects = self.project_service.retrieve_projects_for_user(
member3.id, 1)[0]
assert len(projects) == 2, "Projects not retrieved properly!"
def test_retrieve_projects_random(self):
"""
Generate a large number of users/projects, and validate the results.
"""
ExtremeTestFactory.generate_users(NR_USERS, MAX_PROJ_PER_USER)
for i in range(NR_USERS):
current_user = dao.get_user_by_name("gen" + str(i))
expected_projects = ExtremeTestFactory.VALIDATION_DICT[
current_user.id]
if expected_projects % PROJECTS_PAGE_SIZE == 0:
expected_pages = expected_projects / PROJECTS_PAGE_SIZE
exp_proj_per_page = PROJECTS_PAGE_SIZE
else:
expected_pages = expected_projects // PROJECTS_PAGE_SIZE + 1
exp_proj_per_page = expected_projects % PROJECTS_PAGE_SIZE
if expected_projects == 0:
expected_pages = 0
exp_proj_per_page = 0
projects, pages = self.project_service.retrieve_projects_for_user(
current_user.id, expected_pages)
assert len(projects) == exp_proj_per_page, "Projects not retrieved properly! Expected:" + \
str(exp_proj_per_page) + "but got:" + str(len(projects))
assert pages == expected_pages, "Pages not retrieved properly!"
for folder in os.listdir(TvbProfile.current.TVB_STORAGE):
full_path = os.path.join(TvbProfile.current.TVB_STORAGE, folder)
if os.path.isdir(full_path) and folder.startswith('Generated'):
shutil.rmtree(full_path)
def test_retrieve_projects_page2(self):
"""
Test for retrieving the second page projects for a given user.
"""
for i in range(PROJECTS_PAGE_SIZE + 3):
TestFactory.create_project(self.test_user, 'test_proj' + str(i))
projects, pages = self.project_service.retrieve_projects_for_user(
self.test_user.id, 2)
assert len(projects) == (PROJECTS_PAGE_SIZE + 3
) % PROJECTS_PAGE_SIZE, "Pagination inproper."
assert pages == 2, 'Wrong number of pages retrieved.'
def test_retrieve_projects_and_del(self):
"""
Test for retrieving the second page projects for a given user.
"""
created_projects = []
for i in range(PROJECTS_PAGE_SIZE + 1):
created_projects.append(
TestFactory.create_project(self.test_user,
'test_proj' + str(i)))
projects, pages = self.project_service.retrieve_projects_for_user(
self.test_user.id, 2)
assert len(projects) == (PROJECTS_PAGE_SIZE + 1
) % PROJECTS_PAGE_SIZE, "Pagination improper."
assert pages == (PROJECTS_PAGE_SIZE +
1) // PROJECTS_PAGE_SIZE + 1, 'Wrong number of pages'
self.project_service.remove_project(created_projects[1].id)
projects, pages = self.project_service.retrieve_projects_for_user(
self.test_user.id, 2)
assert len(projects) == 0, "Pagination improper."
assert pages == 1, 'Wrong number of pages retrieved.'
projects, pages = self.project_service.retrieve_projects_for_user(
self.test_user.id, 1)
assert len(projects) == PROJECTS_PAGE_SIZE, "Pagination improper."
assert pages == 1, 'Wrong number of pages retrieved.'
def test_empty_project_has_zero_disk_size(self):
TestFactory.create_project(self.test_user, 'test_proj')
projects, pages = self.project_service.retrieve_projects_for_user(
self.test_user.id)
assert 0 == projects[0].disk_size
assert '0.0 KiB' == projects[0].disk_size_human
def test_project_disk_size(self):
project1 = TestFactory.create_project(self.test_user, 'test_proj1')
zip_path = os.path.join(os.path.dirname(tvb_data.__file__),
'connectivity', 'connectivity_66.zip')
TestFactory.import_zip_connectivity(self.test_user, project1, zip_path,
'testSubject')
project2 = TestFactory.create_project(self.test_user, 'test_proj2')
zip_path = os.path.join(os.path.dirname(tvb_data.__file__),
'connectivity', 'connectivity_76.zip')
TestFactory.import_zip_connectivity(self.test_user, project2, zip_path,
'testSubject')
projects = self.project_service.retrieve_projects_for_user(
self.test_user.id)[0]
assert projects[0].disk_size != projects[
1].disk_size, "projects should have different size"
for project in projects:
assert 0 != project.disk_size
assert '0.0 KiB' != project.disk_size_human
prj_folder = self.structure_helper.get_project_folder(project)
actual_disk_size = FilesHelper.compute_recursive_h5_disk_usage(
prj_folder)
ratio = float(actual_disk_size) / project.disk_size
msg = "Real disk usage: %s The one recorded in the db : %s" % (
actual_disk_size, project.disk_size)
assert ratio < 1.1, msg
def test_get_linkable_projects(self):
"""
Test for retrieving the projects for a given user.
"""
initial_projects = self.project_service.retrieve_projects_for_user(
self.test_user.id)[0]
assert len(initial_projects) == 0, "Database was not reset!"
test_proj = []
user1 = TestFactory.create_user("another_user")
for i in range(4):
test_proj.append(
TestFactory.create_project(self.test_user if i < 3 else user1,
'test_proj' + str(i)))
operation = TestFactory.create_operation(test_user=self.test_user,
test_project=test_proj[0])
datatype = dao.store_entity(
model_datatype.DataType(module="test_data",
subject="subj1",
state="test_state",
operation_id=operation.id))
linkable = self.project_service.get_linkable_projects_for_user(
self.test_user.id, str(datatype.id))[0]
assert len(linkable) == 2, "Wrong count of link-able projects!"
proj_names = [project.name for project in linkable]
assert test_proj[1].name in proj_names
assert test_proj[2].name in proj_names
assert not test_proj[3].name in proj_names
def test_remove_project_happy_flow(self):
"""
Standard flow for deleting a project.
"""
inserted_project = TestFactory.create_project(self.test_user,
'test_proj')
project_root = self.structure_helper.get_project_folder(
inserted_project)
projects = dao.get_projects_for_user(self.test_user.id)
assert len(projects) == 1, "Initializations failed!"
assert os.path.exists(project_root), "Something failed at insert time!"
self.project_service.remove_project(inserted_project.id)
projects = dao.get_projects_for_user(self.test_user.id)
assert len(projects) == 0, "Project was not deleted!"
assert not os.path.exists(project_root), "Root folder not deleted!"
def test_remove_project_wrong_id(self):
"""
Flow for deleting a project giving an un-existing id.
"""
TestFactory.create_project(self.test_user, 'test_proj')
projects = dao.get_projects_for_user(self.test_user.id)
assert len(projects) == 1, "Initializations failed!"
with pytest.raises(ProjectServiceException):
self.project_service.remove_project(99)
def __check_meta_data(self, expected_meta_data, new_datatype):
"""Validate Meta-Data"""
mapp_keys = {
DataTypeOverlayDetails.DATA_SUBJECT: "subject",
DataTypeOverlayDetails.DATA_STATE: "state"
}
for key, value in expected_meta_data.items():
if key in mapp_keys:
assert value == getattr(new_datatype, mapp_keys[key])
elif key == DataTypeMetaData.KEY_OPERATION_TAG:
if DataTypeMetaData.KEY_OP_GROUP_ID in expected_meta_data:
# We have a Group to check
op_group = new_datatype.parent_operation.fk_operation_group
op_group = dao.get_generic_entity(
model_operation.OperationGroup, op_group)[0]
assert value == op_group.name
else:
assert value == new_datatype.parent_operation.user_group
def test_remove_project_node(self):
"""
Test removing of a node from a project.
"""
inserted_project, gid, op = TestFactory.create_value_wrapper(
self.test_user)
project_to_link = model_project.Project("Link", self.test_user.id,
"descript")
project_to_link = dao.store_entity(project_to_link)
exact_data = dao.get_datatype_by_gid(gid)
assert exact_data is not None, "Initialization problem!"
dao.store_entity(
model_datatype.Links(exact_data.id, project_to_link.id))
vw_h5_path = h5.path_for_stored_index(exact_data)
assert os.path.exists(vw_h5_path)
if dao.get_system_user() is None:
dao.store_entity(
model_operation.User(
TvbProfile.current.web.admin.SYSTEM_USER_NAME,
TvbProfile.current.web.admin.SYSTEM_USER_NAME, None, None,
True, None))
self.project_service._remove_project_node_files(
inserted_project.id, gid)
assert not os.path.exists(vw_h5_path)
exact_data = dao.get_datatype_by_gid(gid)
assert exact_data is not None, "Data should still be in DB, because of links"
vw_h5_path_new = h5.path_for_stored_index(exact_data)
assert os.path.exists(vw_h5_path_new)
assert vw_h5_path_new != vw_h5_path
self.project_service._remove_project_node_files(
project_to_link.id, gid)
assert dao.get_datatype_by_gid(gid) is None
def test_update_meta_data_simple(self):
"""
Test the new update metaData for a simple data that is not part of a group.
"""
inserted_project, gid, _ = TestFactory.create_value_wrapper(
self.test_user)
new_meta_data = {
DataTypeOverlayDetails.DATA_SUBJECT: "new subject",
DataTypeOverlayDetails.DATA_STATE: "second_state",
DataTypeOverlayDetails.CODE_GID: gid,
DataTypeOverlayDetails.CODE_OPERATION_TAG: 'new user group'
}
self.project_service.update_metadata(new_meta_data)
new_datatype = dao.get_datatype_by_gid(gid)
self.__check_meta_data(new_meta_data, new_datatype)
new_datatype_h5 = h5.h5_file_for_index(new_datatype)
assert new_datatype_h5.subject.load(
) == 'new subject', 'UserGroup not updated!'
def test_update_meta_data_group(self, test_adapter_factory):
"""
Test the new update metaData for a group of dataTypes.
"""
test_adapter_factory(adapter_class=TestAdapter3)
op_group_id = TestFactory.create_group(test_user=self.test_user)[1]
new_meta_data = {
DataTypeOverlayDetails.DATA_SUBJECT: "new subject",
DataTypeOverlayDetails.DATA_STATE: "updated_state",
DataTypeOverlayDetails.CODE_OPERATION_GROUP_ID: op_group_id,
DataTypeOverlayDetails.CODE_OPERATION_TAG: 'newGroupName'
}
self.project_service.update_metadata(new_meta_data)
datatypes = dao.get_datatype_in_group(op_group_id)
for datatype in datatypes:
new_datatype = dao.get_datatype_by_id(datatype.id)
assert op_group_id == new_datatype.parent_operation.fk_operation_group
new_group = dao.get_generic_entity(model_operation.OperationGroup,
op_group_id)[0]
assert new_group.name == "newGroupName"
self.__check_meta_data(new_meta_data, new_datatype)
def test_retrieve_project_full(self, dummy_datatype_index_factory):
"""
Tests full project information is retrieved by method `ProjectService.retrieve_project_full(...)`
"""
project = TestFactory.create_project(self.test_user)
operation = TestFactory.create_operation(test_user=self.test_user,
test_project=project)
dummy_datatype_index_factory(project=project, operation=operation)
dummy_datatype_index_factory(project=project, operation=operation)
dummy_datatype_index_factory(project=project, operation=operation)
_, ops_nr, operations, pages_no = self.project_service.retrieve_project_full(
project.id)
assert ops_nr == 1, "DataType Factory should only use one operation to store all it's datatypes."
assert pages_no == 1, "DataType Factory should only use one operation to store all it's datatypes."
resulted_dts = operations[0]['results']
assert len(resulted_dts) == 3, "3 datatypes should be created."
def test_get_project_structure(self, datatype_group_factory,
dummy_datatype_index_factory,
project_factory, user_factory):
"""
Tests project structure is as expected and contains all datatypes and created links
"""
user = user_factory()
project1 = project_factory(user, name="TestPS1")
project2 = project_factory(user, name="TestPS2")
dt_group = datatype_group_factory(project=project1)
dt_simple = dummy_datatype_index_factory(state="RAW_DATA",
project=project1)
# Create 3 DTs directly in Project 2
dummy_datatype_index_factory(state="RAW_DATA", project=project2)
dummy_datatype_index_factory(state="RAW_DATA", project=project2)
dummy_datatype_index_factory(state="RAW_DATA", project=project2)
# Create Links from Project 1 into Project 2
link_ids, expected_links = [], []
link_ids.append(dt_simple.id)
expected_links.append(dt_simple.gid)
# Prepare links towards a full DT Group, but expecting only the DT_Group in the final tree
dts = dao.get_datatype_in_group(datatype_group_id=dt_group.id)
link_ids.extend([dt_to_link.id for dt_to_link in dts])
link_ids.append(dt_group.id)
expected_links.append(dt_group.gid)
# Actually create the links from Prj1 into Prj2
AlgorithmService().create_link(link_ids, project2.id)
# Retrieve the raw data used to compose the tree (for easy parsing)
dts_in_tree = dao.get_data_in_project(project2.id)
dts_in_tree = [dt.gid for dt in dts_in_tree]
# Retrieve the tree json (for trivial validations only, as we can not decode)
node_json = self.project_service.get_project_structure(
project2, None, DataTypeMetaData.KEY_STATE,
DataTypeMetaData.KEY_SUBJECT, None)
assert len(expected_links) + 3 == len(
dts_in_tree), "invalid number of nodes in tree"
assert dt_group.gid in dts_in_tree, "DT_Group should be in the Project Tree!"
assert dt_group.gid in node_json, "DT_Group should be in the Project Tree JSON!"
project_dts = dao.get_datatypes_in_project(project2.id)
for dt in project_dts:
if dt.fk_datatype_group is not None:
assert not dt.gid in node_json, "DTs part of a group should not be"
assert not dt.gid in dts_in_tree, "DTs part of a group should not be"
else:
assert dt.gid in node_json, "Simple DTs and DT_Groups should be"
assert dt.gid in dts_in_tree, "Simple DTs and DT_Groups should be"
for link_gid in expected_links:
assert link_gid in node_json, "Expected Link not present"
assert link_gid in dts_in_tree, "Expected Link not present"
class OperationService:
"""
Class responsible for preparing an operation launch.
It will prepare parameters, and decide if the operation is to be executed
immediately, or to be sent on the cluster.
"""
ATT_UID = "uid"
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.file_helper = FilesHelper()
##########################################################################################
######## Methods related to launching operations start here ##############################
##########################################################################################
def initiate_operation(self,
current_user,
project,
adapter_instance,
visible=True,
model_view=None):
"""
Gets the parameters of the computation from the previous inputs form,
and launches a computation (on the cluster or locally).
Invoke custom method on an Adapter Instance. Make sure when the
operation has finished that the correct results are stored into DB.
"""
if not isinstance(adapter_instance, ABCAdapter):
self.logger.warning("Inconsistent Adapter Class:" +
str(adapter_instance.__class__))
raise LaunchException("Developer Exception!!")
algo = adapter_instance.stored_adapter
operation = self.prepare_operation(current_user.id, project, algo,
visible, model_view)
if adapter_instance.launch_mode == AdapterLaunchModeEnum.SYNC_SAME_MEM:
return self.initiate_prelaunch(operation, adapter_instance)
else:
return self._send_to_cluster(operation, adapter_instance,
current_user.username)
@staticmethod
def prepare_metadata(algo_category,
burst=None,
current_ga=GenericAttributes()):
"""
Gather generic_metadata from submitted fields and current to be execute algorithm.
Will populate STATE, GROUP, etc in generic_metadata
"""
generic_metadata = GenericAttributes()
generic_metadata.state = algo_category.defaultdatastate
generic_metadata.parent_burst = burst
generic_metadata.fill_from(current_ga)
return generic_metadata
@staticmethod
def _read_set(values):
""" Parse a committed UI possible list of values, into a set converted into string."""
if isinstance(values, list):
set_values = []
values_str = ""
for val in values:
if val not in set_values:
set_values.append(val)
values_str = values_str + " " + str(val)
values = values_str
return str(values).strip()
def group_operation_launch(self, user_id, project, algorithm_id,
category_id):
"""
Create and prepare the launch of a group of operations.
"""
algorithm = dao.get_algorithm_by_id(algorithm_id)
ops, _ = self.prepare_operation(user_id, project, algorithm)
for operation in ops:
self.launch_operation(operation.id, True)
def _prepare_metric_operation(self, sim_operation):
# type: (Operation) -> Operation
metric_algo = dao.get_algorithm_by_module(MEASURE_METRICS_MODULE,
MEASURE_METRICS_CLASS)
datatype_index = h5.REGISTRY.get_index_for_datatype(TimeSeries)
time_series_index = dao.get_generic_entity(datatype_index,
sim_operation.id,
'fk_from_operation')[0]
ga = self.prepare_metadata(metric_algo.algorithm_category,
time_series_index.fk_parent_burst)
ga.visible = False
view_model = get_class_by_name("{}.{}".format(
MEASURE_METRICS_MODULE, MEASURE_METRICS_MODEL_CLASS))()
view_model.time_series = time_series_index.gid
view_model.algorithms = tuple(ALGORITHMS.keys())
view_model.generic_attributes = ga
parent_burst = dao.get_generic_entity(
BurstConfiguration, time_series_index.fk_parent_burst, 'gid')[0]
metric_op_group = dao.get_operationgroup_by_id(
parent_burst.fk_metric_operation_group)
metric_operation_group_id = parent_burst.fk_metric_operation_group
range_values = sim_operation.range_values
view_model.operation_group_gid = uuid.UUID(metric_op_group.gid)
view_model.ranges = json.dumps(parent_burst.ranges)
view_model.range_values = range_values
view_model.is_metric_operation = True
metric_operation = Operation(view_model.gid.hex,
sim_operation.fk_launched_by,
sim_operation.fk_launched_in,
metric_algo.id,
user_group=ga.operation_tag,
op_group_id=metric_operation_group_id,
range_values=range_values)
metric_operation.visible = False
metric_operation = dao.store_entity(metric_operation)
metrics_datatype_group = dao.get_generic_entity(
DataTypeGroup, metric_operation_group_id, 'fk_operation_group')[0]
if metrics_datatype_group.fk_from_operation is None:
metrics_datatype_group.fk_from_operation = metric_operation.id
dao.store_entity(metrics_datatype_group)
self.store_view_model(metric_operation, sim_operation.project,
view_model)
return metric_operation
@transactional
def prepare_operation(self,
user_id,
project,
algorithm,
visible=True,
view_model=None,
ranges=None,
burst_gid=None,
op_group_id=None):
"""
Do all the necessary preparations for storing an operation. If it's the case of a
range of values create an operation group and multiple operations for each possible
instance from the range.
"""
algo_category = dao.get_category_by_id(algorithm.fk_category)
ga = self.prepare_metadata(algo_category,
current_ga=view_model.generic_attributes,
burst=burst_gid)
ga.visible = visible
view_model.generic_attributes = ga
self.logger.debug("Saving Operation(userId=" + str(user_id) +
",projectId=" + str(project.id) + ",algorithmId=" +
str(algorithm.id) + ")")
operation = Operation(view_model.gid.hex,
user_id,
project.id,
algorithm.id,
user_group=ga.operation_tag,
op_group_id=op_group_id,
range_values=ranges)
operation = dao.store_entity(operation)
self.store_view_model(operation, project, view_model)
return operation
@staticmethod
def store_view_model(operation, project, view_model):
storage_path = FilesHelper().get_project_folder(
project, str(operation.id))
h5.store_view_model(view_model, storage_path)
view_model_size_on_disk = FilesHelper.compute_recursive_h5_disk_usage(
storage_path)
operation.view_model_disk_size = view_model_size_on_disk
dao.store_entity(operation)
def initiate_prelaunch(self, operation, adapter_instance):
"""
Public method.
This should be the common point in calling an adapter- method.
"""
result_msg = ""
temp_files = []
try:
operation = dao.get_operation_by_id(
operation.id) # Load Lazy fields
disk_space_per_user = TvbProfile.current.MAX_DISK_SPACE
pending_op_disk_space = dao.compute_disk_size_for_started_ops(
operation.fk_launched_by)
user_disk_space = dao.compute_user_generated_disk_size(
operation.fk_launched_by) # From kB to Bytes
available_space = disk_space_per_user - pending_op_disk_space - user_disk_space
view_model = adapter_instance.load_view_model(operation)
try:
form = adapter_instance.get_form()
form = form() if isclass(form) else form
fields = form.get_upload_field_names()
project = dao.get_project_by_id(operation.fk_launched_in)
tmp_folder = self.file_helper.get_project_folder(
project, self.file_helper.TEMP_FOLDER)
for upload_field in fields:
if hasattr(view_model, upload_field):
file = getattr(view_model, upload_field)
if file.startswith(tmp_folder) or file.startswith(
TvbProfile.current.TVB_TEMP_FOLDER):
temp_files.append(file)
except AttributeError:
# Skip if we don't have upload fields on current form
pass
result_msg, nr_datatypes = adapter_instance._prelaunch(
operation, view_model, available_space)
operation = dao.get_operation_by_id(operation.id)
operation.mark_complete(STATUS_FINISHED)
dao.store_entity(operation)
self._update_vm_generic_operation_tag(view_model, operation)
self._remove_files(temp_files)
except zipfile.BadZipfile as excep:
msg = "The uploaded file is not a valid ZIP!"
self._handle_exception(excep, temp_files, msg, operation)
except TVBException as excep:
self._handle_exception(excep, temp_files, excep.message, operation)
except MemoryError:
msg = (
"Could not execute operation because there is not enough free memory."
+ " Please adjust operation parameters and re-launch it.")
self._handle_exception(Exception(msg), temp_files, msg, operation)
except Exception as excep1:
msg = "Could not launch Operation with the given input data!"
self._handle_exception(excep1, temp_files, msg, operation)
if operation.fk_operation_group and 'SimulatorAdapter' in operation.algorithm.classname:
next_op = self._prepare_metric_operation(operation)
self.launch_operation(next_op.id)
return result_msg
def _send_to_cluster(self,
operation,
adapter_instance,
current_username="******"):
""" Initiate operation on cluster"""
try:
BackendClientFactory.execute(str(operation.id), current_username,
adapter_instance)
except TVBException as ex:
self._handle_exception(ex, {}, ex.message, operation)
except Exception as excep:
self._handle_exception(excep, {}, "Could not start operation!",
operation)
return operation
@staticmethod
def _update_vm_generic_operation_tag(view_model, operation):
project = dao.get_project_by_id(operation.fk_launched_in)
storage_path = FilesHelper().get_project_folder(
project, str(operation.id))
h5_path = h5.path_for(storage_path, ViewModelH5, view_model.gid,
type(view_model).__name__)
with ViewModelH5(h5_path, view_model) as vm_h5:
vm_h5.operation_tag.store(operation.user_group)
def launch_operation(self,
operation_id,
send_to_cluster=False,
adapter_instance=None):
"""
Method exposed for Burst-Workflow related calls.
It is used for cascading operation in the same workflow.
"""
if operation_id is not None:
operation = dao.get_operation_by_id(operation_id)
if adapter_instance is None:
algorithm = operation.algorithm
adapter_instance = ABCAdapter.build_adapter(algorithm)
if send_to_cluster:
self._send_to_cluster(operation, adapter_instance,
operation.user.username)
else:
self.initiate_prelaunch(operation, adapter_instance)
def _handle_exception(self,
exception,
temp_files,
message,
operation=None):
"""
Common way to treat exceptions:
- remove temporary files, if any
- set status ERROR on current operation (if any)
- log exception
"""
self.logger.exception(message)
if operation is not None:
BurstService().persist_operation_state(operation, STATUS_ERROR,
str(exception))
self._remove_files(temp_files)
exception.message = message
raise exception.with_traceback(
sys.exc_info()[2]
) # when rethrowing in python this is required to preserve the stack trace
def _remove_files(self, file_list):
"""
Remove any files that exist in the file_dictionary.
Currently used to delete temporary files created during an operation.
"""
for pth in file_list:
if pth is not None:
pth = str(pth)
try:
if os.path.exists(pth) and os.path.isfile(pth):
os.remove(pth)
if len(os.listdir(os.path.dirname(pth))) == 0:
shutil.rmtree(os.path.dirname(pth))
self.logger.debug("We no longer need file:" + pth +
" => deleted")
else:
self.logger.warning(
"Trying to remove not existent file:" + pth)
except OSError:
self.logger.exception("Could not cleanup file!")
@staticmethod
def _range_name(range_no):
return PARAM_RANGE_PREFIX + str(range_no)
def fire_operation(self,
adapter_instance,
current_user,
project_id,
visible=True,
view_model=None):
"""
Launch an operation, specified by AdapterInstance, for current_user and project with project_id.
"""
operation_name = str(adapter_instance.__class__.__name__)
try:
self.logger.info("Starting operation " + operation_name)
project = dao.get_project_by_id(project_id)
result = self.initiate_operation(current_user,
project,
adapter_instance,
visible,
model_view=view_model)
self.logger.info("Finished operation launch:" + operation_name)
return result
except TVBException as excep:
self.logger.exception(
"Could not launch operation " + operation_name +
" with the given set of input data, because: " + excep.message)
raise OperationException(excep.message, excep)
except Exception as excep:
self.logger.exception("Could not launch operation " +
operation_name +
" with the given set of input data!")
raise OperationException(str(excep))
@staticmethod
def load_operation(operation_id):
""" Retrieve previously stored Operation from DB, and load operation.burst attribute"""
operation = dao.get_operation_by_id(operation_id)
operation.burst = dao.get_burst_for_operation_id(operation_id)
return operation
@staticmethod
def stop_operation(operation_id, is_group=False, remove_after_stop=False):
# type: (int, bool, bool) -> bool
"""
Stop (also named Cancel) the operation given by operation_id,
and potentially also remove it after (with all linked data).
In case the Operation has a linked Burst, remove that too.
:param operation_id: ID for Operation (or OperationGroup) to be canceled/removed
:param is_group: When true stop all the operations from that group.
:param remove_after_stop: if True, also remove the operation(s) after stopping
:returns True if the stop step was successfully
"""
result = False
if is_group:
op_group = ProjectService.get_operation_group_by_id(operation_id)
operations_in_group = ProjectService.get_operations_in_group(
op_group)
for operation in operations_in_group:
result = OperationService.stop_operation(
operation.id, False, remove_after_stop) or result
elif dao.try_get_operation_by_id(operation_id) is not None:
result = BackendClientFactory.stop_operation(operation_id)
if remove_after_stop:
burst_config = dao.get_burst_for_direct_operation_id(
operation_id)
ProjectService().remove_operation(operation_id)
if burst_config is not None:
result = dao.remove_entity(BurstConfiguration,
burst_config.id) or result
return result
class FlowService:
"""
Service Layer for all TVB generic Work-Flow operations.
"""
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.file_helper = FilesHelper()
self.input_tree_manager = InputTreeManager()
def get_category_by_id(self, identifier):
""" Pass to DAO the retrieve of category by ID operation."""
return dao.get_category_by_id(identifier)
@staticmethod
def get_raw_categories():
""":returns: AlgorithmCategory list of entities that have results in RAW state (Creators/Uploaders)"""
return dao.get_raw_categories()
@staticmethod
def get_visualisers_category():
"""Retrieve all Algorithm categories, with display capability"""
result = dao.get_visualisers_categories()
if not result:
raise ValueError("View Category not found!!!")
return result[0]
@staticmethod
def get_algorithm_by_identifier(ident):
"""
Retrieve Algorithm entity by ID.
Return None, if ID is not found in DB.
"""
return dao.get_algorithm_by_id(ident)
@staticmethod
def load_operation(operation_id):
""" Retrieve previously stored Operation from DB, and load operation.burst attribute"""
operation = dao.get_operation_by_id(operation_id)
operation.burst = dao.get_burst_for_operation_id(operation_id)
return operation
@staticmethod
def get_operation_numbers(proj_id):
""" Count total number of operations started for current project. """
return dao.get_operation_numbers(proj_id)
def prepare_adapter(self, project_id, stored_adapter):
"""
Having a StoredAdapter, return the Tree Adapter Interface object, populated with datatypes from 'project_id'.
"""
adapter_module = stored_adapter.module
adapter_name = stored_adapter.classname
try:
# Prepare Adapter Interface, by populating with existent data,
# in case of a parameter of type DataType.
adapter_instance = ABCAdapter.build_adapter(stored_adapter)
interface = adapter_instance.get_input_tree()
interface = self.input_tree_manager.fill_input_tree_with_options(
interface, project_id, stored_adapter.fk_category)
interface = self.input_tree_manager.prepare_param_names(interface)
return interface
except Exception:
self.logger.exception('Not found:' + adapter_name + ' in:' +
adapter_module)
raise OperationException("Could not prepare " + adapter_name)
@staticmethod
def get_algorithm_by_module_and_class(module, classname):
"""
Get the db entry from the algorithm table for the given module and
class.
"""
return dao.get_algorithm_by_module(module, classname)
@staticmethod
def get_available_datatypes(project_id, data_type_cls, filters=None):
"""
Return all dataTypes that match a given name and some filters.
:param data_type_cls: either a fully qualified class name or a class object
"""
return get_filtered_datatypes(project_id, data_type_cls, filters)
@staticmethod
def create_link(data_ids, project_id):
"""
For a list of dataType IDs and a project id create all the required links.
"""
for data in data_ids:
link = model.Links(data, project_id)
dao.store_entity(link)
@staticmethod
def remove_link(dt_id, project_id):
"""
Remove the link from the datatype given by dt_id to project given by project_id.
"""
link = dao.get_link(dt_id, project_id)
if link is not None:
dao.remove_entity(model.Links, link.id)
def fire_operation(self,
adapter_instance,
current_user,
project_id,
visible=True,
**data):
"""
Launch an operation, specified by AdapterInstance, for CurrentUser,
Current Project and a given set of UI Input Data.
"""
operation_name = str(adapter_instance.__class__.__name__)
try:
self.logger.info("Starting operation " + operation_name)
project = dao.get_project_by_id(project_id)
tmp_folder = self.file_helper.get_project_folder(
project, self.file_helper.TEMP_FOLDER)
result = OperationService().initiate_operation(
current_user, project.id, adapter_instance, tmp_folder,
visible, **data)
self.logger.info("Finished operation:" + operation_name)
return result
except TVBException as excep:
self.logger.exception(
"Could not launch operation " + operation_name +
" with the given set of input data, because: " + excep.message)
raise OperationException(excep.message, excep)
except Exception as excep:
self.logger.exception("Could not launch operation " +
operation_name +
" with the given set of input data!")
raise OperationException(str(excep))
@staticmethod
def get_upload_algorithms():
"""
:return: List of StoredAdapter entities
"""
categories = dao.get_uploader_categories()
categories_ids = [categ.id for categ in categories]
return dao.get_adapters_from_categories(categories_ids)
def get_analyze_groups(self):
"""
:return: list of AlgorithmTransientGroup entities
"""
categories = dao.get_launchable_categories(elimin_viewers=True)
categories_ids = [categ.id for categ in categories]
stored_adapters = dao.get_adapters_from_categories(categories_ids)
groups_list = []
for adapter in stored_adapters:
# For empty groups, this time, we fill the actual adapter
group = AlgorithmTransientGroup(
adapter.group_name or adapter.displayname,
adapter.group_description or adapter.description)
group = self._find_group(groups_list, group)
group.children.append(adapter)
return categories[0], groups_list
@staticmethod
def _find_group(groups_list, new_group):
for i in range(len(groups_list) - 1, -1, -1):
current_group = groups_list[i]
if current_group.name == new_group.name and current_group.description == new_group.description:
return current_group
# Not found in list
groups_list.append(new_group)
return new_group
def get_visualizers_for_group(self, dt_group_gid):
categories = dao.get_visualisers_categories()
return self._get_launchable_algorithms(dt_group_gid, categories)[1]
def get_launchable_algorithms(self, datatype_gid):
"""
:param datatype_gid: Filter only algorithms compatible with this GUID
:return: dict(category_name: List AlgorithmTransientGroup)
"""
categories = dao.get_launchable_categories()
datatype_instance, filtered_adapters = self._get_launchable_algorithms(
datatype_gid, categories)
if isinstance(datatype_instance, model.DataTypeGroup):
# If part of a group, update also with specific analyzers of the child datatype
dt_group = dao.get_datatype_group_by_gid(datatype_gid)
datatypes = dao.get_datatypes_from_datatype_group(dt_group.id)
if len(datatypes):
datatype = datatypes[-1]
analyze_category = dao.get_launchable_categories(True)
_, inner_analyzers = self._get_launchable_algorithms(
datatype.gid, analyze_category)
filtered_adapters.extend(inner_analyzers)
categories_dict = dict()
for c in categories:
categories_dict[c.id] = c.displayname
return self._group_adapters_by_category(filtered_adapters,
categories_dict)
def _get_launchable_algorithms(self, datatype_gid, categories):
datatype_instance = dao.get_datatype_by_gid(datatype_gid)
data_class = datatype_instance.__class__
all_compatible_classes = [data_class.__name__]
for one_class in getmro(data_class):
if issubclass(
one_class, MappedType
) and one_class.__name__ not in all_compatible_classes:
all_compatible_classes.append(one_class.__name__)
self.logger.debug("Searching in categories: " + str(categories) +
" for classes " + str(all_compatible_classes))
categories_ids = [categ.id for categ in categories]
launchable_adapters = dao.get_applicable_adapters(
all_compatible_classes, categories_ids)
filtered_adapters = []
for stored_adapter in launchable_adapters:
filter_chain = FilterChain.from_json(
stored_adapter.datatype_filter)
if not filter_chain or filter_chain.get_python_filter_equivalent(
datatype_instance):
filtered_adapters.append(stored_adapter)
return datatype_instance, filtered_adapters
def _group_adapters_by_category(self, stored_adapters, categories):
"""
:param stored_adapters: list StoredAdapter
:return: dict(category_name: List AlgorithmTransientGroup), empty groups all in the same AlgorithmTransientGroup
"""
categories_dict = dict()
for adapter in stored_adapters:
category_name = categories.get(adapter.fk_category)
if category_name in categories_dict:
groups_list = categories_dict.get(category_name)
else:
groups_list = []
categories_dict[category_name] = groups_list
group = AlgorithmTransientGroup(adapter.group_name,
adapter.group_description)
group = self._find_group(groups_list, group)
group.children.append(adapter)
return categories_dict
@staticmethod
def get_generic_entity(entity_type, filter_value, select_field):
return dao.get_generic_entity(entity_type, filter_value, select_field)
##########################################################################
######## Methods below are for MeasurePoint selections ###################
##########################################################################
@staticmethod
def get_selections_for_project(project_id, datatype_gid):
"""
Retrieved from DB saved selections for current project. If a certain selection
doesn't have all the labels between the labels of the given connectivity than
this selection will not be returned.
:returns: List of ConnectivitySelection entities.
"""
return dao.get_selections_for_project(project_id, datatype_gid)
@staticmethod
def save_measure_points_selection(ui_name, selected_nodes, datatype_gid,
project_id):
"""
Store in DB a ConnectivitySelection.
"""
select_entities = dao.get_selections_for_project(
project_id, datatype_gid, ui_name)
if select_entities:
# when the name of the new selection is within the available selections then update that selection:
select_entity = select_entities[0]
select_entity.selected_nodes = selected_nodes
else:
select_entity = model.MeasurePointsSelection(
ui_name, selected_nodes, datatype_gid, project_id)
dao.store_entity(select_entity)
##########################################################################
########## Bellow are PSE Filters specific methods ##################
##########################################################################
@staticmethod
def get_stored_pse_filters(datatype_group_gid):
return dao.get_stored_pse_filters(datatype_group_gid)
@staticmethod
def save_pse_filter(ui_name, datatype_group_gid, threshold_value,
applied_on):
"""
Store in DB a PSE filter.
"""
select_entities = dao.get_stored_pse_filters(datatype_group_gid,
ui_name)
if select_entities:
# when the UI name is already in DB, update the existing entity
select_entity = select_entities[0]
select_entity.threshold_value = threshold_value
select_entity.applied_on = applied_on # this is the type, as in applied on size or color
else:
select_entity = model.StoredPSEFilter(ui_name, datatype_group_gid,
threshold_value, applied_on)
dao.store_entity(select_entity)
class ExportManager(object):
"""
This class provides basic methods for exporting data types of projects in different formats.
"""
all_exporters = {} # Dictionary containing all available exporters
export_folder = None
EXPORT_FOLDER_NAME = "EXPORT_TMP"
EXPORTED_SIMULATION_NAME = "exported_simulation"
EXPORTED_SIMULATION_DTS_DIR = "datatypes"
ZIP_FILE_EXTENSION = "zip"
logger = get_logger(__name__)
def __init__(self):
# Here we register all available data type exporters
# If new exporters supported, they should be added here
self._register_exporter(TVBExporter())
self._register_exporter(TVBLinkedExporter())
self.export_folder = os.path.join(TvbProfile.current.TVB_STORAGE,
self.EXPORT_FOLDER_NAME)
self.files_helper = FilesHelper()
def _register_exporter(self, exporter):
"""
This method register into an internal format available exporters.
:param exporter: Instance of a data type exporter (extends ABCExporter)
"""
if exporter is not None:
self.all_exporters[exporter.__class__.__name__] = exporter
def get_exporters_for_data(self, data):
"""
Get available exporters for current data type.
:returns: a dictionary with the {exporter_id : label}
"""
if data is None:
raise InvalidExportDataException(
"Could not detect exporters for null data")
self.logger.debug("Trying to determine exporters valid for %s" %
data.type)
results = {}
# No exporter for None data
if data is None:
return results
for exporterId in self.all_exporters.keys():
exporter = self.all_exporters[exporterId]
if exporter.accepts(data):
results[exporterId] = exporter.get_label()
return results
def export_data(self, data, exporter_id, project):
"""
Export provided data using given exporter
:param data: data type to be exported
:param exporter_id: identifier of the exporter to be used
:param project: project that contains data to be exported
:returns: a tuple with the following elements
1. name of the file to be shown to user
2. full path of the export file (available for download)
3. boolean which specify if file can be deleted after download
"""
if data is None:
raise InvalidExportDataException(
"Could not export null data. Please select data to be exported"
)
if exporter_id is None:
raise ExportException(
"Please select the exporter to be used for this operation")
if exporter_id not in self.all_exporters:
raise ExportException(
"Provided exporter identifier is not a valid one")
exporter = self.all_exporters[exporter_id]
if project is None:
raise ExportException(
"Please provide the project where data files are stored")
# Now we start the real export
if not exporter.accepts(data):
raise InvalidExportDataException(
"Current data can not be exported by specified exporter")
# Now compute and create folder where to store exported data
# This will imply to generate a folder which is unique for each export
data_export_folder = None
try:
data_export_folder = self._build_data_export_folder(data)
self.logger.debug("Start export of data: %s" % data.type)
export_data = exporter.export(data, data_export_folder, project)
finally:
# In case export did not generated any file delete folder
if data_export_folder is not None and len(
os.listdir(data_export_folder)) == 0:
os.rmdir(data_export_folder)
return export_data
def _export_linked_datatypes(self, project, zip_file):
linked_paths = ProjectService().get_linked_datatypes_storage_path(
project)
if not linked_paths:
# do not export an empty operation
return
# Make a import operation which will contain links to other projects
algo = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE,
TVB_IMPORTER_CLASS)
op = model_operation.Operation(None, None, project.id, algo.id)
op.project = project
op.algorithm = algo
op.id = 'links-to-external-projects'
op.start_now()
op.mark_complete(model_operation.STATUS_FINISHED)
op_folder = self.files_helper.get_operation_folder(
op.project.name, op.id)
op_folder_name = os.path.basename(op_folder)
# add linked datatypes to archive in the import operation
for pth in linked_paths:
zip_pth = op_folder_name + '/' + os.path.basename(pth)
zip_file.write(pth, zip_pth)
# remove these files, since we only want them in export archive
self.files_helper.remove_folder(op_folder)
def export_project(self, project, optimize_size=False):
"""
Given a project root and the TVB storage_path, create a ZIP
ready for export.
:param project: project object which identifies project to be exported
"""
if project is None:
raise ExportException("Please provide project to be exported")
project_folder = self.files_helper.get_project_folder(project)
project_datatypes = dao.get_datatypes_in_project(
project.id, only_visible=optimize_size)
to_be_exported_folders = []
considered_op_ids = []
folders_to_exclude = self._get_op_with_errors(project.id)
if optimize_size:
# take only the DataType with visibility flag set ON
for dt in project_datatypes:
op_id = dt.fk_from_operation
if op_id not in considered_op_ids:
to_be_exported_folders.append({
'folder':
self.files_helper.get_project_folder(
project, str(op_id)),
'archive_path_prefix':
str(op_id) + os.sep,
'exclude':
folders_to_exclude
})
considered_op_ids.append(op_id)
else:
folders_to_exclude.append("TEMP")
to_be_exported_folders.append({
'folder': project_folder,
'archive_path_prefix': '',
'exclude': folders_to_exclude
})
# Compute path and name of the zip file
now = datetime.now()
date_str = now.strftime("%Y-%m-%d_%H-%M")
zip_file_name = "%s_%s.%s" % (date_str, project.name,
self.ZIP_FILE_EXTENSION)
export_folder = self._build_data_export_folder(project)
result_path = os.path.join(export_folder, zip_file_name)
with TvbZip(result_path, "w") as zip_file:
# Pack project [filtered] content into a ZIP file:
self.logger.debug("Done preparing, now we will write folders " +
str(len(to_be_exported_folders)))
self.logger.debug(str(to_be_exported_folders))
for pack in to_be_exported_folders:
zip_file.write_folder(**pack)
self.logger.debug(
"Done exporting files, now we will export linked DTs")
self._export_linked_datatypes(project, zip_file)
# Make sure the Project.xml file gets copied:
if optimize_size:
self.logger.debug("Done linked, now we write the project xml")
zip_file.write(
self.files_helper.get_project_meta_file_path(project.name),
self.files_helper.TVB_PROJECT_FILE)
self.logger.debug("Done, closing")
return result_path
@staticmethod
def _get_op_with_errors(project_id):
"""
Get the operation folders with error base name as list.
"""
operations = dao.get_operations_with_error_in_project(project_id)
op_with_errors = []
for op in operations:
op_with_errors.append(op.id)
return op_with_errors
def _build_data_export_folder(self, data):
"""
This method computes the folder where results of an export operation will be
stored for a while (e.g until download is done; or for 1 day)
"""
now = datetime.now()
date_str = "%d-%d-%d_%d-%d-%d_%d" % (now.year, now.month, now.day,
now.hour, now.minute, now.second,
now.microsecond)
tmp_str = date_str + "@" + data.gid
data_export_folder = os.path.join(self.export_folder, tmp_str)
self.files_helper.check_created(data_export_folder)
return data_export_folder
def export_simulator_configuration(self, burst_id):
burst = dao.get_burst_by_id(burst_id)
if burst is None:
raise InvalidExportDataException("Could not find burst with ID " +
str(burst_id))
op_folder = self.files_helper.get_project_folder(
burst.project, str(burst.fk_simulation))
tmp_export_folder = self._build_data_export_folder(burst)
tmp_sim_folder = os.path.join(tmp_export_folder,
self.EXPORTED_SIMULATION_NAME)
if not os.path.exists(tmp_sim_folder):
os.makedirs(tmp_sim_folder)
all_view_model_paths, all_datatype_paths = h5.gather_references_of_view_model(
burst.simulator_gid, op_folder)
burst_path = h5.determine_filepath(burst.gid, op_folder)
all_view_model_paths.append(burst_path)
for vm_path in all_view_model_paths:
dest = os.path.join(tmp_sim_folder, os.path.basename(vm_path))
self.files_helper.copy_file(vm_path, dest)
for dt_path in all_datatype_paths:
dest = os.path.join(tmp_sim_folder,
self.EXPORTED_SIMULATION_DTS_DIR,
os.path.basename(dt_path))
self.files_helper.copy_file(dt_path, dest)
main_vm_path = h5.determine_filepath(burst.simulator_gid,
tmp_sim_folder)
H5File.remove_metadata_param(main_vm_path, 'history_gid')
now = datetime.now()
date_str = now.strftime("%Y-%m-%d_%H-%M")
zip_file_name = "%s_%s.%s" % (date_str, str(burst_id),
self.ZIP_FILE_EXTENSION)
result_path = os.path.join(tmp_export_folder, zip_file_name)
with TvbZip(result_path, "w") as zip_file:
zip_file.write_folder(tmp_sim_folder)
self.files_helper.remove_folder(tmp_sim_folder)
return result_path
class DatatypesFactory(object):
"""
This class provides a set of methods that helps user to create
different data types for testing.
These data types will be automatically stored in DB and file system if needed.
"""
USER_FULL_NAME = "Datatype Factory User"
DATATYPE_STATE = "RAW_DATA"
DATATYPE_DATA = ["test", "for", "datatypes", "factory"]
DATATYPE_MEASURE_METRIC = {'v': 3}
RANGE_1 = ["row1", [1, 2, 3]]
RANGE_2 = ["row2", [0.1, 0.3, 0.5]]
user = None
project = None
operation = None
def __init__(self):
micro_postfix = "_%d" % int(time.time() * 1000000)
# Here create all structures needed later for data types creation
self.files_helper = FilesHelper()
# First create user
user = model.User("datatype_factory_user" + micro_postfix, "test_pass",
"*****@*****.**" + micro_postfix, True, "user")
self.user = dao.store_entity(user)
# Now create a project
project_service = ProjectService()
data = dict(name='DatatypesFactoryProject' + micro_postfix, description='test_desc', users=[])
self.project = project_service.store_project(self.user, True, None, **data)
# Create algorithm
alg_category = model.AlgorithmCategory('one', True)
dao.store_entity(alg_category)
ad = model.Algorithm("test_module1", "classname1", alg_category.id)
self.algorithm = dao.store_entity(ad)
#Create an operation
self.meta = {DataTypeMetaData.KEY_SUBJECT: self.USER_FULL_NAME,
DataTypeMetaData.KEY_STATE: self.DATATYPE_STATE}
operation = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters',
meta=json.dumps(self.meta), status=model.STATUS_FINISHED)
self.operation = dao.store_entity(operation)
def get_project(self):
"""
Return project to which generated data types are assigned
"""
return self.project
def get_operation(self):
"""
Return operation to which generated data types are assigned
"""
return self.operation
def get_user(self):
"""
Return user to which generated data types are assigned
"""
return self.user
def _store_datatype(self, data_type, operation_id=None):
"""
Launch adapter to store a create a persistent DataType.
"""
operation_id = operation_id or self.operation.id
data_type.type = data_type.__class__.__name__
data_type.module = data_type.__class__.__module__
data_type.subject = self.USER_FULL_NAME
data_type.state = self.DATATYPE_STATE
data_type.set_operation_id(operation_id)
adapter_instance = StoreAdapter([data_type])
operation = dao.get_operation_by_id(operation_id)
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return data_type
def create_simple_datatype(self, subject=USER_FULL_NAME, state=DATATYPE_STATE):
"""
This method creates a simple data type
"""
datatype_inst = Datatype1()
self._fill_datatype(datatype_inst, subject, state)
# Store data type
return self._store_datatype(datatype_inst)
def create_datatype_with_storage(self, subject=USER_FULL_NAME, state=DATATYPE_STATE,
data=DATATYPE_DATA, operation_id=None):
"""
This method creates and stores a data type which imply storage on the file system.
"""
datatype_inst = Datatype2()
self._fill_datatype(datatype_inst, subject, state, operation_id)
datatype_inst.string_data = data
return self._store_datatype(datatype_inst, operation_id)
def _fill_datatype(self, datatype, subject, state, operation_id=None):
"""
This method sets some common attributes on dataType
"""
operation_id = operation_id or self.operation.id
datatype.subject = subject
datatype.state = state
# Set_operation_id also sets storage_path attribute
datatype.set_operation_id(operation_id)
def __create_operation(self):
"""
Create a operation entity. Return the operation, algo_id and the storage path.
"""
meta = {DataTypeMetaData.KEY_SUBJECT: "John Doe", DataTypeMetaData.KEY_STATE: "RAW_DATA"}
algorithm = FlowService().get_algorithm_by_module_and_class(SIMULATOR_MODULE, SIMULATOR_CLASS)
operation = model.Operation(self.user.id, self.project.id, algorithm.id, json.dumps(''), meta=json.dumps(meta),
status=model.STATUS_STARTED)
operation = dao.store_entity(operation)
storage_path = FilesHelper().get_project_folder(self.project, str(operation.id))
return operation, algorithm.id, storage_path
def create_connectivity(self, nodes=74):
"""
Create a connectivity that will be used in "non-dummy" burst launches (with the actual simulator).
"""
operation, algo_id, storage_path = self.__create_operation()
connectivity = Connectivity(storage_path=storage_path)
connectivity.weights = numpy.ones((nodes, nodes))
connectivity.centres = numpy.ones((nodes, 3))
adapter_instance = StoreAdapter([connectivity])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return algo_id, connectivity
def create_timeseries(self, connectivity, ts_type=None, sensors=None):
"""
Create a stored TimeSeries entity.
"""
operation, _, storage_path = self.__create_operation()
if ts_type == "EEG":
time_series = TimeSeriesEEG(storage_path=storage_path, sensors=sensors)
else:
rm = dao.get_generic_entity(RegionMapping, connectivity.gid, '_connectivity')
if len(rm) < 1:
rm = None
else:
rm = rm[0]
time_series = TimeSeriesRegion(storage_path=storage_path, connectivity=connectivity, region_mapping=rm)
data = numpy.random.random((10, 10, 10, 10))
time_series.write_data_slice(data)
time_series.write_time_slice(numpy.arange(10))
adapter_instance = StoreAdapter([time_series])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
time_series = dao.get_datatype_by_gid(time_series.gid)
return time_series
def create_stimulus(self, connectivity):
"""
:param connectivity: Connectivity to create stimuli for its regions
:return: persisted region Stimuli instance
"""
operation, _, storage_path = self.__create_operation()
stimuli_region = StimuliRegion(storage_path=storage_path)
stimuli_region.connectivity = connectivity
stimuli_region.weight = numpy.random.random((connectivity.number_of_regions, 1)).tolist()
stimuli_region.temporal = PulseTrain()
adapter_instance = StoreAdapter([stimuli_region])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return stimuli_region
def create_covariance(self, time_series):
"""
:returns: a stored DataType Covariance.
"""
operation, _, storage_path = self.__create_operation()
covariance = Covariance(storage_path=storage_path, source=time_series)
covariance.write_data_slice(numpy.random.random((10, 10, 10)))
adapter_instance = StoreAdapter([covariance])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return covariance
def create_crosscoherence(self, time_series):
"""
:returns: a stored entity of type CoherenceSpectrum
"""
operation, _, storage_path = self.__create_operation()
partial_coh = CoherenceSpectrum(array_data=numpy.random.random((10, 10, 10, 10)), use_storage=False)
coherence = CoherenceSpectrum(source=time_series, storage_path=storage_path, frequency=0.1, nfft=256)
coherence.write_data_slice(partial_coh)
coherence.close_file()
adapter_instance = StoreAdapter([coherence])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return coherence
def create_crosscorrelation(self, time_series):
"""
:returns: `CrossCorrelation` stored entity.
"""
operation, _, storage_path = self.__create_operation()
partial_corr = CrossCorrelation(array_data=numpy.random.random((10, 10, 10, 10, 10)), use_storage=False)
crossc = CrossCorrelation(source=time_series, storage_path=storage_path, time=range(10))
crossc.write_data_slice(partial_corr)
crossc.close_file()
adapter_instance = StoreAdapter([crossc])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return crossc
def create_surface(self):
"""
Create a dummy surface entity.
:returns: (Algorithm Identifier, stored Surface entity)
"""
operation, algo_id, storage_path = self.__create_operation()
surface = CorticalSurface(storage_path=storage_path)
surface.vertices = numpy.array([[-10, 0, 0],
[0, 0, -10],
[10, 0, 0],
[0, 10, 0]], dtype=float)
surface.triangles = numpy.array([[0, 1, 2],
[0, 1, 3],
[1, 2, 3],
[0, 2, 3]], dtype=int)
surface.number_of_triangles = 4
surface.number_of_vertices = 4
surface.triangle_normals = numpy.ones((4, 3))
surface.vertex_normals = numpy.ones((4, 3))
surface.zero_based_triangles = True
surface.validate()
adapter_instance = StoreAdapter([surface])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return algo_id, surface
def create_connectivity_measure(self, connectivity):
"""
:returns: persisted entity ConnectivityMeasure
"""
operation, _, storage_path = self.__create_operation()
conn_measure = ConnectivityMeasure(storage_path=storage_path)
conn_measure.connectivity = connectivity
adapter_instance = StoreAdapter([conn_measure])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return conn_measure
def create_datatype_measure(self, analyzed_entity, operation=None, storage_path=None):
"""
:return: persisted DatatypeMeasure
"""
if operation is None:
operation, _, storage_path = self.__create_operation()
measure = DatatypeMeasure(storage_path=storage_path, metrics=self.DATATYPE_MEASURE_METRIC)
measure.analyzed_datatype = analyzed_entity
adapter_instance = StoreAdapter([measure])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return measure
def create_ICA(self, timeseries):
"""
:returns: persisted entity IndependentComponents
"""
operation, _, storage_path = self.__create_operation()
partial_ts = TimeSeries(use_storage=False)
partial_ts.data = numpy.random.random((10, 10, 10, 10))
partial_ica = IndependentComponents(source=partial_ts,
component_time_series=numpy.random.random((10, 10, 10, 10)),
prewhitening_matrix=numpy.random.random((10, 10, 10, 10)),
unmixing_matrix=numpy.random.random((10, 10, 10, 10)),
n_components=10, use_storage=False)
ica = IndependentComponents(source=timeseries, n_components=10, storage_path=storage_path)
ica.write_data_slice(partial_ica)
adapter_instance = StoreAdapter([ica])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
return ica
def create_datatype_group(self, subject=USER_FULL_NAME, state=DATATYPE_STATE, ):
"""
This method creates, stores and returns a DataTypeGroup entity.
"""
group = model.OperationGroup(self.project.id, ranges=[json.dumps(self.RANGE_1), json.dumps(self.RANGE_2)])
group = dao.store_entity(group)
group_ms = model.OperationGroup(self.project.id, ranges=[json.dumps(self.RANGE_1), json.dumps(self.RANGE_2)])
group_ms = dao.store_entity(group_ms)
datatype_group = model.DataTypeGroup(group, subject=subject, state=state, operation_id=self.operation.id)
# Set storage path, before setting data
datatype_group.storage_path = self.files_helper.get_project_folder(self.project, str(self.operation.id))
datatype_group = dao.store_entity(datatype_group)
dt_group_ms = model.DataTypeGroup(group_ms, subject=subject, state=state, operation_id=self.operation.id)
# Set storage path, before setting data
dt_group_ms.storage_path = self.files_helper.get_project_folder(self.project, str(self.operation.id))
dao.store_entity(dt_group_ms)
# Now create some data types and add them to group
for range_val1 in self.RANGE_1[1]:
for range_val2 in self.RANGE_2[1]:
operation = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters',
meta=json.dumps(self.meta), status=model.STATUS_FINISHED,
range_values=json.dumps({self.RANGE_1[0]: range_val1,
self.RANGE_2[0]: range_val2}))
operation.fk_operation_group = group.id
operation = dao.store_entity(operation)
datatype = self.create_datatype_with_storage(operation_id=operation.id)
datatype.number1 = range_val1
datatype.number2 = range_val2
datatype.fk_datatype_group = datatype_group.id
datatype.set_operation_id(operation.id)
dao.store_entity(datatype)
op_ms = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters',
meta=json.dumps(self.meta), status=model.STATUS_FINISHED,
range_values=json.dumps({self.RANGE_1[0]: range_val1,
self.RANGE_2[0]: range_val2}))
op_ms.fk_operation_group = group_ms.id
op_ms = dao.store_entity(op_ms)
self.create_datatype_measure(datatype, op_ms,
FilesHelper().get_project_folder(self.project, str(op_ms.id)))
return datatype_group
class FlowService:
"""
Service Layer for all TVB generic Work-Flow operations.
"""
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.file_helper = FilesHelper()
self.input_tree_manager = InputTreeManager()
def get_category_by_id(self, identifier):
""" Pass to DAO the retrieve of category by ID operation."""
return dao.get_category_by_id(identifier)
@staticmethod
def get_raw_categories():
""":returns: AlgorithmCategory list of entities that have results in RAW state (Creators/Uploaders)"""
return dao.get_raw_categories()
@staticmethod
def get_visualisers_category():
"""Retrieve all Algorithm categories, with display capability"""
result = dao.get_visualisers_categories()
if not result:
raise ValueError("View Category not found!!!")
return result[0]
@staticmethod
def get_algorithm_by_identifier(ident):
"""
Retrieve Algorithm entity by ID.
Return None, if ID is not found in DB.
"""
return dao.get_algorithm_by_id(ident)
@staticmethod
def load_operation(operation_id):
""" Retrieve previously stored Operation from DB, and load operation.burst attribute"""
operation = dao.get_operation_by_id(operation_id)
operation.burst = dao.get_burst_for_operation_id(operation_id)
return operation
@staticmethod
def get_operation_numbers(proj_id):
""" Count total number of operations started for current project. """
return dao.get_operation_numbers(proj_id)
def prepare_adapter(self, project_id, stored_adapter):
"""
Having a StoredAdapter, return the Tree Adapter Interface object, populated with datatypes from 'project_id'.
"""
adapter_module = stored_adapter.module
adapter_name = stored_adapter.classname
try:
# Prepare Adapter Interface, by populating with existent data,
# in case of a parameter of type DataType.
adapter_instance = ABCAdapter.build_adapter(stored_adapter)
interface = adapter_instance.get_input_tree()
interface = self.input_tree_manager.fill_input_tree_with_options(interface, project_id, stored_adapter.fk_category)
interface = self.input_tree_manager.prepare_param_names(interface)
return interface
except Exception:
self.logger.exception('Not found:' + adapter_name + ' in:' + adapter_module)
raise OperationException("Could not prepare " + adapter_name)
@staticmethod
def get_algorithm_by_module_and_class(module, classname):
"""
Get the db entry from the algorithm table for the given module and
class.
"""
return dao.get_algorithm_by_module(module, classname)
@staticmethod
def get_available_datatypes(project_id, data_type_cls, filters=None):
"""
Return all dataTypes that match a given name and some filters.
:param data_type_cls: either a fully qualified class name or a class object
"""
return get_filtered_datatypes(project_id, data_type_cls, filters)
@staticmethod
def create_link(data_ids, project_id):
"""
For a list of dataType IDs and a project id create all the required links.
"""
for data in data_ids:
link = model.Links(data, project_id)
dao.store_entity(link)
@staticmethod
def remove_link(dt_id, project_id):
"""
Remove the link from the datatype given by dt_id to project given by project_id.
"""
link = dao.get_link(dt_id, project_id)
if link is not None:
dao.remove_entity(model.Links, link.id)
def fire_operation(self, adapter_instance, current_user, project_id, visible=True, **data):
"""
Launch an operation, specified by AdapterInstance, for CurrentUser,
Current Project and a given set of UI Input Data.
"""
operation_name = str(adapter_instance.__class__.__name__)
try:
self.logger.info("Starting operation " + operation_name)
project = dao.get_project_by_id(project_id)
tmp_folder = self.file_helper.get_project_folder(project, self.file_helper.TEMP_FOLDER)
result = OperationService().initiate_operation(current_user, project.id, adapter_instance,
tmp_folder, visible, **data)
self.logger.info("Finished operation launch:" + operation_name)
return result
except TVBException as excep:
self.logger.exception("Could not launch operation " + operation_name +
" with the given set of input data, because: " + excep.message)
raise OperationException(excep.message, excep)
except Exception as excep:
self.logger.exception("Could not launch operation " + operation_name + " with the given set of input data!")
raise OperationException(str(excep))
@staticmethod
def get_upload_algorithms():
"""
:return: List of StoredAdapter entities
"""
categories = dao.get_uploader_categories()
categories_ids = [categ.id for categ in categories]
return dao.get_adapters_from_categories(categories_ids)
def get_analyze_groups(self):
"""
:return: list of AlgorithmTransientGroup entities
"""
categories = dao.get_launchable_categories(elimin_viewers=True)
categories_ids = [categ.id for categ in categories]
stored_adapters = dao.get_adapters_from_categories(categories_ids)
groups_list = []
for adapter in stored_adapters:
# For empty groups, this time, we fill the actual adapter
group = AlgorithmTransientGroup(adapter.group_name or adapter.displayname,
adapter.group_description or adapter.description)
group = self._find_group(groups_list, group)
group.children.append(adapter)
return categories[0], groups_list
@staticmethod
def _find_group(groups_list, new_group):
for i in range(len(groups_list) - 1, -1, -1):
current_group = groups_list[i]
if current_group.name == new_group.name and current_group.description == new_group.description:
return current_group
# Not found in list
groups_list.append(new_group)
return new_group
def get_visualizers_for_group(self, dt_group_gid):
categories = dao.get_visualisers_categories()
return self._get_launchable_algorithms(dt_group_gid, categories)[1]
def get_launchable_algorithms(self, datatype_gid):
"""
:param datatype_gid: Filter only algorithms compatible with this GUID
:return: dict(category_name: List AlgorithmTransientGroup)
"""
categories = dao.get_launchable_categories()
datatype_instance, filtered_adapters = self._get_launchable_algorithms(datatype_gid, categories)
if isinstance(datatype_instance, model.DataTypeGroup):
# If part of a group, update also with specific analyzers of the child datatype
dt_group = dao.get_datatype_group_by_gid(datatype_gid)
datatypes = dao.get_datatypes_from_datatype_group(dt_group.id)
if len(datatypes):
datatype = datatypes[-1]
analyze_category = dao.get_launchable_categories(True)
_, inner_analyzers = self._get_launchable_algorithms(datatype.gid, analyze_category)
filtered_adapters.extend(inner_analyzers)
categories_dict = dict()
for c in categories:
categories_dict[c.id] = c.displayname
return self._group_adapters_by_category(filtered_adapters, categories_dict)
def _get_launchable_algorithms(self, datatype_gid, categories):
datatype_instance = dao.get_datatype_by_gid(datatype_gid)
data_class = datatype_instance.__class__
all_compatible_classes = [data_class.__name__]
for one_class in getmro(data_class):
if issubclass(one_class, MappedType) and one_class.__name__ not in all_compatible_classes:
all_compatible_classes.append(one_class.__name__)
self.logger.debug("Searching in categories: " + str(categories) + " for classes " + str(all_compatible_classes))
categories_ids = [categ.id for categ in categories]
launchable_adapters = dao.get_applicable_adapters(all_compatible_classes, categories_ids)
filtered_adapters = []
for stored_adapter in launchable_adapters:
filter_chain = FilterChain.from_json(stored_adapter.datatype_filter)
if not filter_chain or filter_chain.get_python_filter_equivalent(datatype_instance):
filtered_adapters.append(stored_adapter)
return datatype_instance, filtered_adapters
def _group_adapters_by_category(self, stored_adapters, categories):
"""
:param stored_adapters: list StoredAdapter
:return: dict(category_name: List AlgorithmTransientGroup), empty groups all in the same AlgorithmTransientGroup
"""
categories_dict = dict()
for adapter in stored_adapters:
category_name = categories.get(adapter.fk_category)
if category_name in categories_dict:
groups_list = categories_dict.get(category_name)
else:
groups_list = []
categories_dict[category_name] = groups_list
group = AlgorithmTransientGroup(adapter.group_name, adapter.group_description)
group = self._find_group(groups_list, group)
group.children.append(adapter)
return categories_dict
@staticmethod
def get_generic_entity(entity_type, filter_value, select_field):
return dao.get_generic_entity(entity_type, filter_value, select_field)
##########################################################################
######## Methods below are for MeasurePoint selections ###################
##########################################################################
@staticmethod
def get_selections_for_project(project_id, datatype_gid):
"""
Retrieved from DB saved selections for current project. If a certain selection
doesn't have all the labels between the labels of the given connectivity than
this selection will not be returned.
:returns: List of ConnectivitySelection entities.
"""
return dao.get_selections_for_project(project_id, datatype_gid)
@staticmethod
def save_measure_points_selection(ui_name, selected_nodes, datatype_gid, project_id):
"""
Store in DB a ConnectivitySelection.
"""
select_entities = dao.get_selections_for_project(project_id, datatype_gid, ui_name)
if select_entities:
# when the name of the new selection is within the available selections then update that selection:
select_entity = select_entities[0]
select_entity.selected_nodes = selected_nodes
else:
select_entity = model.MeasurePointsSelection(ui_name, selected_nodes, datatype_gid, project_id)
dao.store_entity(select_entity)
##########################################################################
########## Bellow are PSE Filters specific methods ##################
##########################################################################
@staticmethod
def get_stored_pse_filters(datatype_group_gid):
return dao.get_stored_pse_filters(datatype_group_gid)
@staticmethod
def save_pse_filter(ui_name, datatype_group_gid, threshold_value, applied_on):
"""
Store in DB a PSE filter.
"""
select_entities = dao.get_stored_pse_filters(datatype_group_gid, ui_name)
if select_entities:
# when the UI name is already in DB, update the existing entity
select_entity = select_entities[0]
select_entity.threshold_value = threshold_value
select_entity.applied_on = applied_on # this is the type, as in applied on size or color
else:
select_entity = model.StoredPSEFilter(ui_name, datatype_group_gid, threshold_value, applied_on)
dao.store_entity(select_entity)
class OperationService:
"""
Class responsible for preparing an operation launch.
It will prepare parameters, and decide if the operation is to be executed
immediately, or to be sent on the cluster.
"""
ATT_UID = "uid"
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.file_helper = FilesHelper()
##########################################################################################
######## Methods related to launching operations start here ##############################
##########################################################################################
def initiate_operation(self,
current_user,
project,
adapter_instance,
visible=True,
model_view=None,
**kwargs):
"""
Gets the parameters of the computation from the previous inputs form,
and launches a computation (on the cluster or locally).
Invoke custom method on an Adapter Instance. Make sure when the
operation has finished that the correct results are stored into DB.
"""
if not isinstance(adapter_instance, ABCAdapter):
self.logger.warning("Inconsistent Adapter Class:" +
str(adapter_instance.__class__))
raise LaunchException("Developer Exception!!")
# Store Operation entity.
algo = adapter_instance.stored_adapter
algo_category = dao.get_category_by_id(algo.fk_category)
operations = self.prepare_operations(current_user.id,
project,
algo,
algo_category, {},
visible,
view_model=model_view,
**kwargs)[0]
if isinstance(adapter_instance, ABCSynchronous):
if len(operations) > 1:
raise LaunchException(
"Synchronous operations are not supporting ranges!")
if len(operations) < 1:
self.logger.warning("No operation was defined")
raise LaunchException("Invalid empty Operation!!!")
return self.initiate_prelaunch(operations[0], adapter_instance,
**kwargs)
else:
return self._send_to_cluster(operations, adapter_instance,
current_user.username)
@staticmethod
def _prepare_metadata(initial_metadata, algo_category, operation_group,
submit_data):
"""
Gather metadata from submitted fields and current to be execute algorithm.
Will populate STATE, GROUP in metadata
"""
metadata = copy(initial_metadata)
user_group = None
if DataTypeMetaData.KEY_OPERATION_TAG in submit_data:
user_group = submit_data[DataTypeMetaData.KEY_OPERATION_TAG]
if operation_group is not None:
metadata[DataTypeMetaData.KEY_OPERATION_TAG] = operation_group.name
if DataTypeMetaData.KEY_TAG_1 in submit_data:
metadata[DataTypeMetaData.KEY_TAG_1] = submit_data[
DataTypeMetaData.KEY_TAG_1]
metadata[DataTypeMetaData.KEY_STATE] = algo_category.defaultdatastate
return metadata, user_group
@staticmethod
def _read_set(values):
""" Parse a committed UI possible list of values, into a set converted into string."""
if isinstance(values, list):
set_values = []
values_str = ""
for val in values:
if val not in set_values:
set_values.append(val)
values_str = values_str + " " + str(val)
values = values_str
return str(values).strip()
def group_operation_launch(self,
user_id,
project,
algorithm_id,
category_id,
existing_dt_group=None,
**kwargs):
"""
Create and prepare the launch of a group of operations.
"""
category = dao.get_category_by_id(category_id)
algorithm = dao.get_algorithm_by_id(algorithm_id)
ops, _ = self.prepare_operations(user_id,
project,
algorithm,
category, {},
existing_dt_group=existing_dt_group,
**kwargs)
for operation in ops:
self.launch_operation(operation.id, True)
def _prepare_metric_operation(self, sim_operation):
# type: (Operation) -> None
metric_algo = dao.get_algorithm_by_module(
TimeseriesMetricsAdapter.__module__,
TimeseriesMetricsAdapter.__name__)
time_series_index = dao.get_generic_entity(TimeSeriesIndex,
sim_operation.id,
'fk_from_operation')[0]
view_model = TimeseriesMetricsAdapterModel()
view_model.time_series = time_series_index.gid
view_model.algorithms = tuple(choices.values())
range_values = sim_operation.range_values
metadata = {
DataTypeMetaData.KEY_BURST: time_series_index.fk_parent_burst
}
metadata, user_group = self._prepare_metadata(
metadata, metric_algo.algorithm_category, None, {})
meta_str = json.dumps(metadata)
parent_burst = dao.get_generic_entity(
BurstConfiguration, time_series_index.fk_parent_burst, 'id')[0]
metric_operation_group_id = parent_burst.fk_metric_operation_group
metric_operation = Operation(sim_operation.fk_launched_by,
sim_operation.fk_launched_in,
metric_algo.id,
json.dumps({'gid': view_model.gid.hex}),
meta_str,
op_group_id=metric_operation_group_id,
range_values=range_values)
metric_operation.visible = False
stored_metric_operation = dao.store_entity(metric_operation)
metrics_datatype_group = dao.get_generic_entity(
DataTypeGroup, metric_operation_group_id, 'fk_operation_group')[0]
if metrics_datatype_group.fk_from_operation is None:
metrics_datatype_group.fk_from_operation = metric_operation.id
OperationService._store_view_model(stored_metric_operation,
sim_operation.project, view_model)
return stored_metric_operation
def prepare_operation(self,
user_id,
project_id,
algorithm_id,
category,
view_model_gid,
op_group,
metadata,
ranges=None,
visible=True):
operation_parameters = json.dumps({'gid': view_model_gid})
metadata, user_group = self._prepare_metadata(metadata, category,
op_group, {})
meta_str = json.dumps(metadata)
op_group_id = None
if op_group:
op_group_id = op_group.id
operation = Operation(user_id,
project_id,
algorithm_id,
operation_parameters,
op_group_id=op_group_id,
meta=meta_str,
range_values=ranges)
self.logger.debug("Saving Operation(userId=" + str(user_id) +
",projectId=" + str(project_id) + "," +
str(metadata) + ",algorithmId=" + str(algorithm_id) +
", ops_group= " + str(op_group_id) + ")")
visible_operation = visible and category.display is False
operation = dao.store_entity(operation)
operation.visible = visible_operation
return operation
def prepare_operations(self,
user_id,
project,
algorithm,
category,
metadata,
visible=True,
existing_dt_group=None,
view_model=None,
**kwargs):
"""
Do all the necessary preparations for storing an operation. If it's the case of a
range of values create an operation group and multiple operations for each possible
instance from the range.
:param metadata: Initial MetaData with potential Burst identification inside.
"""
# TODO: fix group operations
operations = []
available_args, group = self._prepare_group(project.id,
existing_dt_group, kwargs)
if len(available_args) > TvbProfile.current.MAX_RANGE_NUMBER:
raise LaunchException(
"Too big range specified. You should limit the"
" resulting operations to %d" %
TvbProfile.current.MAX_RANGE_NUMBER)
else:
self.logger.debug("Launching a range with %d operations..." %
len(available_args))
group_id = None
if group is not None:
group_id = group.id
metadata, user_group = self._prepare_metadata(metadata, category,
group, kwargs)
self.logger.debug("Saving Operation(userId=" + str(user_id) +
",projectId=" + str(project.id) + "," +
str(metadata) + ",algorithmId=" + str(algorithm.id) +
", ops_group= " + str(group_id) + ")")
visible_operation = visible and category.display is False
meta_str = json.dumps(metadata)
for (one_set_of_args, range_vals) in available_args:
range_values = json.dumps(range_vals) if range_vals else None
operation = Operation(user_id,
project.id,
algorithm.id,
json.dumps({'gid': view_model.gid.hex}),
meta_str,
op_group_id=group_id,
user_group=user_group,
range_values=range_values)
operation.visible = visible_operation
operations.append(operation)
operations = dao.store_entities(operations)
if group is not None:
burst_id = None
if DataTypeMetaData.KEY_BURST in metadata:
burst_id = metadata[DataTypeMetaData.KEY_BURST]
if existing_dt_group is None:
datatype_group = DataTypeGroup(
group,
operation_id=operations[0].id,
fk_parent_burst=burst_id,
state=metadata[DataTypeMetaData.KEY_STATE])
dao.store_entity(datatype_group)
else:
# Reset count
existing_dt_group.count_results = None
dao.store_entity(existing_dt_group)
for operation in operations:
OperationService._store_view_model(operation, project, view_model)
return operations, group
@staticmethod
def _store_view_model(operation, project, view_model):
storage_path = FilesHelper().get_project_folder(
project, str(operation.id))
h5_path = h5.path_for(storage_path, ViewModelH5, view_model.gid)
h5_file = ViewModelH5(h5_path, view_model)
h5_file.store(view_model)
h5_file.close()
def load_view_model(self, adapter_instance, operation):
storage_path = self.file_helper.get_project_folder(
operation.project, str(operation.id))
input_gid = json.loads(operation.parameters)['gid']
# TODO: review location, storage_path, op params deserialization
if isinstance(adapter_instance, SimulatorAdapter):
view_model = SimulatorSerializer().deserialize_simulator(
input_gid, storage_path)
else:
view_model_class = adapter_instance.get_view_model_class()
view_model = view_model_class()
h5_path = h5.path_for(storage_path, ViewModelH5, input_gid)
h5_file = ViewModelH5(h5_path, view_model)
h5_file.load_into(view_model)
return view_model
def initiate_prelaunch(self, operation, adapter_instance, **kwargs):
"""
Public method.
This should be the common point in calling an adapter- method.
"""
result_msg = ""
temp_files = []
try:
unique_id = None
if self.ATT_UID in kwargs:
unique_id = kwargs[self.ATT_UID]
operation = dao.get_operation_by_id(
operation.id) # Load Lazy fields
disk_space_per_user = TvbProfile.current.MAX_DISK_SPACE
pending_op_disk_space = dao.compute_disk_size_for_started_ops(
operation.fk_launched_by)
user_disk_space = dao.compute_user_generated_disk_size(
operation.fk_launched_by) # From kB to Bytes
available_space = disk_space_per_user - pending_op_disk_space - user_disk_space
view_model = self.load_view_model(adapter_instance, operation)
result_msg, nr_datatypes = adapter_instance._prelaunch(
operation, unique_id, available_space, view_model=view_model)
operation = dao.get_operation_by_id(operation.id)
## Update DB stored kwargs for search purposes, to contain only valuable params (no unselected options)
operation.parameters = json.dumps(kwargs)
operation.mark_complete(STATUS_FINISHED)
if nr_datatypes > 0:
#### Write operation meta-XML only if some result are returned
self.file_helper.write_operation_metadata(operation)
dao.store_entity(operation)
adapter_form = adapter_instance.get_form()
try:
temp_files = adapter_form.temporary_files
except AttributeError:
pass
self._remove_files(temp_files)
except zipfile.BadZipfile as excep:
msg = "The uploaded file is not a valid ZIP!"
self._handle_exception(excep, temp_files, msg, operation)
except TVBException as excep:
self._handle_exception(excep, temp_files, excep.message, operation)
except MemoryError:
msg = (
"Could not execute operation because there is not enough free memory."
+ " Please adjust operation parameters and re-launch it.")
self._handle_exception(Exception(msg), temp_files, msg, operation)
except Exception as excep1:
msg = "Could not launch Operation with the given input data!"
self._handle_exception(excep1, temp_files, msg, operation)
if operation.fk_operation_group and 'SimulatorAdapter' in operation.algorithm.classname:
next_op = self._prepare_metric_operation(operation)
self.launch_operation(next_op.id)
return result_msg
def _send_to_cluster(self,
operations,
adapter_instance,
current_username="******"):
""" Initiate operation on cluster"""
for operation in operations:
try:
BACKEND_CLIENT.execute(str(operation.id), current_username,
adapter_instance)
except Exception as excep:
self._handle_exception(excep, {}, "Could not start operation!",
operation)
return operations
def launch_operation(self,
operation_id,
send_to_cluster=False,
adapter_instance=None):
"""
Method exposed for Burst-Workflow related calls.
It is used for cascading operation in the same workflow.
"""
if operation_id is not None:
operation = dao.get_operation_by_id(operation_id)
if adapter_instance is None:
algorithm = operation.algorithm
adapter_instance = ABCAdapter.build_adapter(algorithm)
if send_to_cluster:
self._send_to_cluster([operation], adapter_instance,
operation.user.username)
else:
self.initiate_prelaunch(operation, adapter_instance)
def _handle_exception(self,
exception,
temp_files,
message,
operation=None):
"""
Common way to treat exceptions:
- remove temporary files, if any
- set status ERROR on current operation (if any)
- log exception
"""
self.logger.exception(message)
if operation is not None:
BurstService().persist_operation_state(operation, STATUS_ERROR,
str(exception))
self._remove_files(temp_files)
exception.message = message
raise exception.with_traceback(
sys.exc_info()[2]
) # when rethrowing in python this is required to preserve the stack trace
def _remove_files(self, file_list):
"""
Remove any files that exist in the file_dictionary.
Currently used to delete temporary files created during an operation.
"""
for pth in file_list:
pth = str(pth)
try:
if os.path.exists(pth) and os.path.isfile(pth):
os.remove(pth)
self.logger.debug("We no longer need file:" + pth +
" => deleted")
else:
self.logger.warning("Trying to remove not existent file:" +
pth)
except OSError:
self.logger.exception("Could not cleanup file!")
@staticmethod
def _range_name(range_no):
return PARAM_RANGE_PREFIX + str(range_no)
def _prepare_group(self, project_id, existing_dt_group, kwargs):
"""
Create and store OperationGroup entity, or return None
"""
# Standard ranges as accepted from UI
range1_values = self.get_range_values(kwargs, self._range_name(1))
range2_values = self.get_range_values(kwargs, self._range_name(2))
available_args = self.__expand_arguments([(kwargs, None)],
range1_values,
self._range_name(1))
available_args = self.__expand_arguments(available_args, range2_values,
self._range_name(2))
is_group = False
ranges = []
if self._range_name(1) in kwargs and range1_values is not None:
is_group = True
ranges.append(
json.dumps((kwargs[self._range_name(1)], range1_values)))
if self._range_name(2) in kwargs and range2_values is not None:
is_group = True
ranges.append(
json.dumps((kwargs[self._range_name(2)], range2_values)))
# Now for additional ranges which might be the case for the 'model exploration'
last_range_idx = 3
ranger_name = self._range_name(last_range_idx)
while ranger_name in kwargs:
values_for_range = self.get_range_values(kwargs, ranger_name)
available_args = self.__expand_arguments(available_args,
values_for_range,
ranger_name)
last_range_idx += 1
ranger_name = self._range_name(last_range_idx)
if last_range_idx > 3:
ranges = [
] # Since we only have 3 fields in db for this just hide it
if not is_group:
group = None
elif existing_dt_group is None:
group = OperationGroup(project_id=project_id, ranges=ranges)
group = dao.store_entity(group)
else:
group = existing_dt_group.parent_operation_group
return available_args, group
def get_range_values(self, kwargs, ranger_name):
"""
For the ranger given by ranger_name look in kwargs and return
the array with all the possible values.
"""
if ranger_name not in kwargs:
return None
if str(kwargs[ranger_name]) not in kwargs:
return None
range_values = []
try:
range_data = json.loads(str(kwargs[str(kwargs[ranger_name])]))
except Exception:
try:
range_data = [
x.strip()
for x in str(kwargs[str(kwargs[ranger_name])]).split(',')
if len(x.strip()) > 0
]
return range_data
except Exception:
self.logger.exception("Could not launch operation !")
raise LaunchException("Could not launch with no data from:" +
str(ranger_name))
if type(range_data) in (list, tuple):
return range_data
if (constants.ATT_MINVALUE in range_data) and (constants.ATT_MAXVALUE
in range_data):
lo_val = float(range_data[constants.ATT_MINVALUE])
hi_val = float(range_data[constants.ATT_MAXVALUE])
step = float(range_data[constants.ATT_STEP])
range_values = list(
Range(
lo=lo_val, hi=hi_val,
step=step).to_array()) # , mode=Range.MODE_INCLUDE_BOTH))
else:
for possible_value in range_data:
if range_data[possible_value]:
range_values.append(possible_value)
return range_values
@staticmethod
def __expand_arguments(arguments_list, range_values, range_title):
"""
Parse the arguments submitted from UI (flatten form)
If any ranger is found, return a list of arguments for all possible operations.
"""
if range_values is None:
return arguments_list
result = []
for value in range_values:
for args, range_ in arguments_list:
kw_new = copy(args)
range_new = copy(range_)
kw_new[kw_new[range_title]] = value
if range_new is None:
range_new = {}
range_new[kw_new[range_title]] = value
del kw_new[range_title]
result.append((kw_new, range_new))
return result
##########################################################################################
######## Methods related to stopping and restarting operations start here ################
##########################################################################################
def stop_operation(self, operation_id):
"""
Stop the operation given by the operation id.
"""
return BACKEND_CLIENT.stop_operation(int(operation_id))
class ProjectServiceTest(TransactionalTestCase):
"""
This class contains tests for the tvb.core.services.project_service module.
"""
def setUp(self):
"""
Reset the database before each test.
"""
config.EVENTS_FOLDER = ''
self.project_service = ProjectService()
self.structure_helper = FilesHelper()
self.test_user = TestFactory.create_user()
def tearDown(self):
"""
Remove project folders and clean up database.
"""
created_projects = dao.get_projects_for_user(self.test_user.id)
for project in created_projects:
self.structure_helper.remove_project_structure(project.name)
self.delete_project_folders()
def test_create_project_happy_flow(self):
"""
Standard flow for creating a new project.
"""
user1 = TestFactory.create_user('test_user1')
user2 = TestFactory.create_user('test_user2')
initial_projects = dao.get_projects_for_user(self.test_user.id)
self.assertEqual(len(initial_projects), 0, "Database reset probably failed!")
TestFactory.create_project(self.test_user, 'test_project', users=[user1.id, user2.id])
resulting_projects = dao.get_projects_for_user(self.test_user.id)
self.assertEqual(len(resulting_projects), 1, "Project with valid data not inserted!")
project = resulting_projects[0]
if project.name == "test_project":
self.assertEqual(project.description, "description", "Description do no match")
users_for_project = dao.get_members_of_project(project.id)
for user in users_for_project:
self.assertTrue(user.id in [user1.id, user2.id], "Users not stored properly.")
self.assertTrue(os.path.exists(os.path.join(TvbProfile.current.TVB_STORAGE, FilesHelper.PROJECTS_FOLDER,
"test_project")), "Folder for project was not created")
def test_create_project_empty_name(self):
"""
Creating a project with an empty name.
"""
data = dict(name="", description="test_description", users=[])
initial_projects = dao.get_projects_for_user(self.test_user.id)
self.assertEqual(len(initial_projects), 0, "Database reset probably failed!")
self.assertRaises(ProjectServiceException, self.project_service.store_project,
self.test_user, True, None, **data)
def test_edit_project_happy_flow(self):
"""
Standard flow for editing an existing project.
"""
selected_project = TestFactory.create_project(self.test_user, 'test_proj')
proj_root = self.structure_helper.get_project_folder(selected_project)
initial_projects = dao.get_projects_for_user(self.test_user.id)
self.assertEqual(len(initial_projects), 1, "Database initialization probably failed!")
edited_data = dict(name="test_project", description="test_description", users=[])
edited_project = self.project_service.store_project(self.test_user, False, selected_project.id, **edited_data)
self.assertFalse(os.path.exists(proj_root), "Previous folder not deleted")
proj_root = self.structure_helper.get_project_folder(edited_project)
self.assertTrue(os.path.exists(proj_root), "New folder not created!")
self.assertNotEqual(selected_project.name, edited_project.name, "Project was no changed!")
def test_edit_project_unexisting(self):
"""
Trying to edit an un-existing project.
"""
selected_project = TestFactory.create_project(self.test_user, 'test_proj')
self.structure_helper.get_project_folder(selected_project)
initial_projects = dao.get_projects_for_user(self.test_user.id)
self.assertEqual(len(initial_projects), 1, "Database initialization probably failed!")
data = dict(name="test_project", description="test_description", users=[])
self.assertRaises(ProjectServiceException, self.project_service.store_project,
self.test_user, False, 99, **data)
def test_find_project_happy_flow(self):
"""
Standard flow for finding a project by it's id.
"""
initial_projects = dao.get_projects_for_user(self.test_user.id)
self.assertEqual(len(initial_projects), 0, "Database reset probably failed!")
inserted_project = TestFactory.create_project(self.test_user, 'test_project')
self.assertTrue(self.project_service.find_project(inserted_project.id) is not None, "Project not found !")
dao_returned_project = dao.get_project_by_id(inserted_project.id)
service_returned_project = self.project_service.find_project(inserted_project.id)
self.assertEqual(dao_returned_project.id, service_returned_project.id,
"Data returned from service is different from data returned by DAO.")
self.assertEqual(dao_returned_project.name, service_returned_project.name,
"Data returned from service is different than data returned by DAO.")
self.assertEqual(dao_returned_project.description, service_returned_project.description,
"Data returned from service is different from data returned by DAO.")
self.assertEqual(dao_returned_project.members, service_returned_project.members,
"Data returned from service is different from data returned by DAO.")
def test_find_project_unexisting(self):
"""
Searching for an un-existing project.
"""
data = dict(name="test_project", description="test_description", users=[])
initial_projects = dao.get_projects_for_user(self.test_user.id)
self.assertEqual(len(initial_projects), 0, "Database reset probably failed!")
self.project_service.store_project(self.test_user, True, None, **data)
self.assertRaises(ProjectServiceException, self.project_service.find_project, 99)
def test_retrieve_projects_for_user(self):
"""
Test for retrieving the projects for a given user. One page only.
"""
initial_projects = self.project_service.retrieve_projects_for_user(self.test_user.id)[0]
self.assertEqual(len(initial_projects), 0, "Database was not reset properly!")
TestFactory.create_project(self.test_user, 'test_proj')
TestFactory.create_project(self.test_user, 'test_proj1')
TestFactory.create_project(self.test_user, 'test_proj2')
user1 = TestFactory.create_user('another_user')
TestFactory.create_project(user1, 'test_proj3')
projects = self.project_service.retrieve_projects_for_user(self.test_user.id)[0]
self.assertEqual(len(projects), 3, "Projects not retrieved properly!")
for project in projects:
self.assertNotEquals(project.name, "test_project3", "This project should not have been retrieved")
def test_retrieve_1project_3usr(self):
"""
One user as admin, two users as members, getting projects for admin and for any of
the members should return one.
"""
member1 = TestFactory.create_user("member1")
member2 = TestFactory.create_user("member2")
TestFactory.create_project(self.test_user, 'Testproject', users=[member1.id, member2.id])
projects = self.project_service.retrieve_projects_for_user(self.test_user.id, 1)[0]
self.assertEqual(len(projects), 1, "Projects not retrieved properly!")
projects = self.project_service.retrieve_projects_for_user(member1.id, 1)[0]
self.assertEqual(len(projects), 1, "Projects not retrieved properly!")
projects = self.project_service.retrieve_projects_for_user(member2.id, 1)[0]
self.assertEqual(len(projects), 1, "Projects not retrieved properly!")
def test_retrieve_3projects_3usr(self):
"""
Three users, 3 projects. Structure of db:
proj1: {admin: user1, members: [user2, user3]}
proj2: {admin: user2, members: [user1]}
proj3: {admin: user3, members: [user1, user2]}
Check valid project returns for all the users.
"""
member1 = TestFactory.create_user("member1")
member2 = TestFactory.create_user("member2")
member3 = TestFactory.create_user("member3")
TestFactory.create_project(member1, 'TestProject1', users=[member2.id, member3.id])
TestFactory.create_project(member2, 'TestProject2', users=[member1.id])
TestFactory.create_project(member3, 'TestProject3', users=[member1.id, member2.id])
projects = self.project_service.retrieve_projects_for_user(member1.id, 1)[0]
self.assertEqual(len(projects), 3, "Projects not retrieved properly!")
projects = self.project_service.retrieve_projects_for_user(member2.id, 1)[0]
self.assertEqual(len(projects), 3, "Projects not retrieved properly!")
projects = self.project_service.retrieve_projects_for_user(member3.id, 1)[0]
self.assertEqual(len(projects), 2, "Projects not retrieved properly!")
def test_retrieve_projects_random(self):
"""
Generate a large number of users/projects, and validate the results.
"""
ExtremeTestFactory.generate_users(NR_USERS, MAX_PROJ_PER_USER)
for i in range(NR_USERS):
current_user = dao.get_user_by_name("gen" + str(i))
expected_projects = ExtremeTestFactory.VALIDATION_DICT[current_user.id]
if expected_projects % PROJECTS_PAGE_SIZE == 0:
expected_pages = expected_projects / PROJECTS_PAGE_SIZE
exp_proj_per_page = PROJECTS_PAGE_SIZE
else:
expected_pages = expected_projects / PROJECTS_PAGE_SIZE + 1
exp_proj_per_page = expected_projects % PROJECTS_PAGE_SIZE
if expected_projects == 0:
expected_pages = 0
exp_proj_per_page = 0
projects, pages = self.project_service.retrieve_projects_for_user(current_user.id, expected_pages)
self.assertEqual(len(projects), exp_proj_per_page, "Projects not retrieved properly! Expected:" +
str(exp_proj_per_page) + "but got:" + str(len(projects)))
self.assertEqual(pages, expected_pages, "Pages not retrieved properly!")
for folder in os.listdir(TvbProfile.current.TVB_STORAGE):
full_path = os.path.join(TvbProfile.current.TVB_STORAGE, folder)
if os.path.isdir(full_path) and folder.startswith('Generated'):
shutil.rmtree(full_path)
def test_retrieve_projects_page2(self):
"""
Test for retrieving the second page projects for a given user.
"""
for i in range(PROJECTS_PAGE_SIZE + 3):
TestFactory.create_project(self.test_user, 'test_proj' + str(i))
projects, pages = self.project_service.retrieve_projects_for_user(self.test_user.id, 2)
self.assertEqual(len(projects), (PROJECTS_PAGE_SIZE + 3) % PROJECTS_PAGE_SIZE, "Pagination inproper.")
self.assertEqual(pages, 2, 'Wrong number of pages retrieved.')
def test_retrieve_projects_and_del(self):
"""
Test for retrieving the second page projects for a given user.
"""
created_projects = []
for i in range(PROJECTS_PAGE_SIZE + 1):
created_projects.append(TestFactory.create_project(self.test_user, 'test_proj' + str(i)))
projects, pages = self.project_service.retrieve_projects_for_user(self.test_user.id, 2)
self.assertEqual(len(projects), (PROJECTS_PAGE_SIZE + 1) % PROJECTS_PAGE_SIZE, "Pagination improper.")
self.assertEqual(pages, (PROJECTS_PAGE_SIZE + 1) / PROJECTS_PAGE_SIZE + 1, 'Wrong number of pages')
self.project_service.remove_project(created_projects[1].id)
projects, pages = self.project_service.retrieve_projects_for_user(self.test_user.id, 2)
self.assertEqual(len(projects), 0, "Pagination improper.")
self.assertEqual(pages, 1, 'Wrong number of pages retrieved.')
projects, pages = self.project_service.retrieve_projects_for_user(self.test_user.id, 1)
self.assertEqual(len(projects), PROJECTS_PAGE_SIZE, "Pagination improper.")
self.assertEqual(pages, 1, 'Wrong number of pages retrieved.')
def test_empty_project_has_zero_disk_size(self):
TestFactory.create_project(self.test_user, 'test_proj')
projects, pages = self.project_service.retrieve_projects_for_user(self.test_user.id)
self.assertEqual(0, projects[0].disk_size)
self.assertEqual('0.0 KiB', projects[0].disk_size_human)
def test_project_disk_size(self):
project1 = TestFactory.create_project(self.test_user, 'test_proj1')
zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_66.zip')
TestFactory.import_zip_connectivity(self.test_user, project1, 'testSubject', zip_path)
project2 = TestFactory.create_project(self.test_user, 'test_proj2')
TestFactory.import_cff(test_user=self.test_user, test_project=project2)
projects = self.project_service.retrieve_projects_for_user(self.test_user.id)[0]
self.assertNotEqual(projects[0].disk_size, projects[1].disk_size, "projects should have different size")
for project in projects:
self.assertNotEqual(0, project.disk_size)
self.assertNotEqual('0.0 KiB', project.disk_size_human)
prj_folder = self.structure_helper.get_project_folder(project)
actual_disk_size = self.compute_recursive_h5_disk_usage(prj_folder)[0]
ratio = float(actual_disk_size) / project.disk_size
msg = "Real disk usage: %s The one recorded in the db : %s" % (actual_disk_size, project.disk_size)
self.assertTrue(ratio < 1.4, msg)
def test_get_linkable_projects(self):
"""
Test for retrieving the projects for a given user.
"""
initial_projects = self.project_service.retrieve_projects_for_user(self.test_user.id)[0]
self.assertEqual(len(initial_projects), 0, "Database was not reset!")
test_proj = []
user1 = TestFactory.create_user("another_user")
for i in range(4):
test_proj.append(TestFactory.create_project(self.test_user if i < 3 else user1, 'test_proj' + str(i)))
project_storage = self.structure_helper.get_project_folder(test_proj[0])
operation = TestFactory.create_operation(test_user=self.test_user, test_project=test_proj[0])
project_storage = os.path.join(project_storage, str(operation.id))
os.makedirs(project_storage)
datatype = dao.store_entity(model.DataType(module="test_data", subject="subj1",
state="test_state", operation_id=operation.id))
linkable = self.project_service.get_linkable_projects_for_user(self.test_user.id, str(datatype.id))[0]
self.assertEqual(len(linkable), 2, "Wrong count of link-able projects!")
proj_names = [project.name for project in linkable]
self.assertTrue(test_proj[1].name in proj_names)
self.assertTrue(test_proj[2].name in proj_names)
self.assertFalse(test_proj[3].name in proj_names)
def test_remove_project_happy_flow(self):
"""
Standard flow for deleting a project.
"""
inserted_project = TestFactory.create_project(self.test_user, 'test_proj')
project_root = self.structure_helper.get_project_folder(inserted_project)
projects = dao.get_projects_for_user(self.test_user.id)
self.assertEqual(len(projects), 1, "Initializations failed!")
self.assertTrue(os.path.exists(project_root), "Something failed at insert time!")
self.project_service.remove_project(inserted_project.id)
projects = dao.get_projects_for_user(self.test_user.id)
self.assertEqual(len(projects), 0, "Project was not deleted!")
self.assertFalse(os.path.exists(project_root), "Root folder not deleted!")
def test_remove_project_wrong_id(self):
"""
Flow for deleting a project giving an un-existing id.
"""
TestFactory.create_project(self.test_user, 'test_proj')
projects = dao.get_projects_for_user(self.test_user.id)
self.assertEqual(len(projects), 1, "Initializations failed!")
self.assertRaises(ProjectServiceException, self.project_service.remove_project, 99)
@staticmethod
def _create_value_wrapper(test_user, test_project=None):
"""
Creates a ValueWrapper dataType, and the associated parent Operation.
This is also used in ProjectStructureTest.
"""
if test_project is None:
test_project = TestFactory.create_project(test_user, 'test_proj')
operation = TestFactory.create_operation(test_user=test_user, test_project=test_project)
value_wrapper = ValueWrapper(data_value=5.0, data_name="my_value")
value_wrapper.type = "ValueWrapper"
value_wrapper.module = "tvb.datatypes.mapped_values"
value_wrapper.subject = "John Doe"
value_wrapper.state = "RAW_STATE"
value_wrapper.set_operation_id(operation.id)
adapter_instance = StoreAdapter([value_wrapper])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
all_value_wrappers = FlowService().get_available_datatypes(test_project.id,
"tvb.datatypes.mapped_values.ValueWrapper")[0]
if len(all_value_wrappers) != 1:
raise Exception("Should be only one value wrapper.")
result_vw = ABCAdapter.load_entity_by_gid(all_value_wrappers[0][2])
return test_project, result_vw.gid, operation.gid
def __check_meta_data(self, expected_meta_data, new_datatype):
"""Validate Meta-Data"""
mapp_keys = {DataTypeMetaData.KEY_SUBJECT: "subject", DataTypeMetaData.KEY_STATE: "state"}
for key, value in expected_meta_data.iteritems():
if key in mapp_keys:
self.assertEqual(value, getattr(new_datatype, mapp_keys[key]))
elif key == DataTypeMetaData.KEY_OPERATION_TAG:
if DataTypeMetaData.KEY_OP_GROUP_ID in expected_meta_data:
## We have a Group to check
op_group = new_datatype.parent_operation.fk_operation_group
op_group = dao.get_generic_entity(model.OperationGroup, op_group)[0]
self.assertEqual(value, op_group.name)
else:
self.assertEqual(value, new_datatype.parent_operation.user_group)
def test_remove_project_node(self):
"""
Test removing of a node from a project.
"""
inserted_project, gid, gid_op = self._create_value_wrapper(self.test_user)
project_to_link = model.Project("Link", self.test_user.id, "descript")
project_to_link = dao.store_entity(project_to_link)
exact_data = dao.get_datatype_by_gid(gid)
dao.store_entity(model.Links(exact_data.id, project_to_link.id))
self.assertTrue(dao.get_datatype_by_gid(gid) is not None, "Initialization problem!")
operation_id = dao.get_generic_entity(model.Operation, gid_op, 'gid')[0].id
op_folder = self.structure_helper.get_project_folder("test_proj", str(operation_id))
self.assertTrue(os.path.exists(op_folder))
sub_files = os.listdir(op_folder)
self.assertEqual(2, len(sub_files))
### Validate that no more files are created than needed.
self.project_service._remove_project_node_files(inserted_project.id, gid)
sub_files = os.listdir(op_folder)
self.assertEqual(1, len(sub_files))
### operation.xml file should still be there
op_folder = self.structure_helper.get_project_folder("Link", str(operation_id + 1))
sub_files = os.listdir(op_folder)
self.assertEqual(2, len(sub_files))
self.assertTrue(dao.get_datatype_by_gid(gid) is not None, "Data should still be in DB, because of links")
self.project_service._remove_project_node_files(project_to_link.id, gid)
self.assertTrue(dao.get_datatype_by_gid(gid) is None)
sub_files = os.listdir(op_folder)
self.assertEqual(1, len(sub_files))
### operation.xml file should still be there
def test_update_meta_data_simple(self):
"""
Test the new update metaData for a simple data that is not part of a group.
"""
inserted_project, gid, _ = self._create_value_wrapper(self.test_user)
new_meta_data = {DataTypeOverlayDetails.DATA_SUBJECT: "new subject",
DataTypeOverlayDetails.DATA_STATE: "second_state",
DataTypeOverlayDetails.CODE_GID: gid,
DataTypeOverlayDetails.CODE_OPERATION_TAG: 'new user group'}
self.project_service.update_metadata(new_meta_data)
new_datatype = dao.get_datatype_by_gid(gid)
self.__check_meta_data(new_meta_data, new_datatype)
op_path = FilesHelper().get_operation_meta_file_path(inserted_project.name, new_datatype.parent_operation.id)
op_meta = XMLReader(op_path).read_metadata()
self.assertEqual(op_meta['user_group'], 'new user group', 'UserGroup not updated!')
def test_update_meta_data_group(self):
"""
Test the new update metaData for a group of dataTypes.
"""
datatypes, group_id = TestFactory.create_group(self.test_user, subject="test-subject-1")
new_meta_data = {DataTypeOverlayDetails.DATA_SUBJECT: "new subject",
DataTypeOverlayDetails.DATA_STATE: "updated_state",
DataTypeOverlayDetails.CODE_OPERATION_GROUP_ID: group_id,
DataTypeOverlayDetails.CODE_OPERATION_TAG: 'newGroupName'}
self.project_service.update_metadata(new_meta_data)
for datatype in datatypes:
new_datatype = dao.get_datatype_by_id(datatype.id)
self.assertEqual(group_id, new_datatype.parent_operation.fk_operation_group)
new_group = dao.get_generic_entity(model.OperationGroup, group_id)[0]
self.assertEqual(new_group.name, "newGroupName")
self.__check_meta_data(new_meta_data, new_datatype)
def _create_datatypes(self, dt_factory, nr_of_dts):
for idx in range(nr_of_dts):
dt = Datatype1()
dt.row1 = "value%i" % (idx,)
dt.row2 = "value%i" % (idx + 1,)
dt_factory._store_datatype(dt)
def test_retrieve_project_full(self):
"""
Tests full project information is retrieved by method `ProjectService.retrieve_project_full(...)`
"""
dt_factory = datatypes_factory.DatatypesFactory()
self._create_datatypes(dt_factory, 3)
_, ops_nr, operations, pages_no = self.project_service.retrieve_project_full(dt_factory.project.id)
self.assertEqual(ops_nr, 1, "DataType Factory should only use one operation to store all it's datatypes.")
self.assertEqual(pages_no, 1, "DataType Factory should only use one operation to store all it's datatypes.")
resulted_dts = operations[0]['results']
self.assertEqual(len(resulted_dts), 3, "3 datatypes should be created.")
def test_get_project_structure(self):
"""
Tests project structure is as expected and contains all datatypes
"""
SELF_DTS_NUMBER = 3
dt_factory_1 = datatypes_factory.DatatypesFactory()
self._create_datatypes(dt_factory_1, SELF_DTS_NUMBER)
dt_group = dt_factory_1.create_datatype_group()
link_ids, expected_links = [], []
# Prepare link towards a simple DT
dt_factory_2 = datatypes_factory.DatatypesFactory()
dt_to_link = dt_factory_2.create_simple_datatype()
link_ids.append(dt_to_link.id)
expected_links.append(dt_to_link.gid)
# Prepare links towards a full DT Group, but expecting only the DT_Group in the final tree
link_gr = dt_factory_2.create_datatype_group()
dts = dao.get_datatype_in_group(datatype_group_id=link_gr.id)
link_ids.extend([dt_to_link.id for dt_to_link in dts])
link_ids.append(link_gr.id)
expected_links.append(link_gr.gid)
# Prepare link towards a single DT inside a group, and expecting to find the DT in the final tree
link_gr = dt_factory_2.create_datatype_group()
dt_to_link = dao.get_datatype_in_group(datatype_group_id=link_gr.id)[0]
link_ids.append(dt_to_link.id)
expected_links.append(dt_to_link.gid)
# Actually create the links from Prj2 into Prj1
FlowService().create_link(link_ids, dt_factory_1.project.id)
# Retrieve the raw data used to compose the tree (for easy parsing)
dts_in_tree = dao.get_data_in_project(dt_factory_1.project.id)
dts_in_tree = [dt.gid for dt in dts_in_tree]
# Retrieve the tree json (for trivial validations only, as we can not decode)
node_json = self.project_service.get_project_structure(dt_factory_1.project, None, DataTypeMetaData.KEY_STATE,
DataTypeMetaData.KEY_SUBJECT, None)
self.assertEqual(len(expected_links) + SELF_DTS_NUMBER + 2, len(dts_in_tree), "invalid number of nodes in tree")
self.assertFalse(link_gr.gid in dts_in_tree, "DT_group where a single DT is linked is not expected.")
self.assertTrue(dt_group.gid in dts_in_tree, "DT_Group should be in the Project Tree!")
self.assertTrue(dt_group.gid in node_json, "DT_Group should be in the Project Tree JSON!")
project_dts = dao.get_datatypes_in_project(dt_factory_1.project.id)
for dt in project_dts:
if dt.fk_datatype_group is not None:
self.assertFalse(dt.gid in node_json, "DTs part of a group should not be")
self.assertFalse(dt.gid in dts_in_tree, "DTs part of a group should not be")
else:
self.assertTrue(dt.gid in node_json, "Simple DTs and DT_Groups should be")
self.assertTrue(dt.gid in dts_in_tree, "Simple DTs and DT_Groups should be")
for link_gid in expected_links:
self.assertTrue(link_gid in node_json, "Expected Link not present")
self.assertTrue(link_gid in dts_in_tree, "Expected Link not present")
class TestHPCSchedulerClient(BaseTestCase):
def setup_method(self):
self.files_helper = FilesHelper()
self.encryption_handler = EncryptionHandler('123')
self.clean_database()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user)
def _prepare_dummy_files(self, tmpdir):
dummy_file1 = os.path.join(str(tmpdir), 'dummy1.txt')
open(dummy_file1, 'a').close()
dummy_file2 = os.path.join(str(tmpdir), 'dummy2.txt')
open(dummy_file2, 'a').close()
job_inputs = [dummy_file1, dummy_file2]
return job_inputs
def test_encrypt_inputs(self, tmpdir):
job_inputs = self._prepare_dummy_files(tmpdir)
job_encrypted_inputs = self.encryption_handler.encrypt_inputs(
job_inputs)
# Encrypted folder has 2 more files are more then plain folder
assert len(job_encrypted_inputs) == len(job_inputs)
def test_decrypt_results(self, tmpdir):
# Prepare encrypted dir
job_inputs = self._prepare_dummy_files(tmpdir)
self.encryption_handler.encrypt_inputs(job_inputs)
encrypted_dir = self.encryption_handler.get_encrypted_dir()
# Unencrypt data
out_dir = os.path.join(str(tmpdir), 'output')
os.mkdir(out_dir)
self.encryption_handler.decrypt_results_to_dir(out_dir)
list_plain_dir = os.listdir(out_dir)
assert len(list_plain_dir) == len(os.listdir(encrypted_dir))
assert 'dummy1.txt' in list_plain_dir
assert 'dummy2.txt' in list_plain_dir
def test_decrypt_files(self, tmpdir):
# Prepare encrypted dir
job_inputs = self._prepare_dummy_files(tmpdir)
enc_files = self.encryption_handler.encrypt_inputs(job_inputs)
# Unencrypt data
out_dir = os.path.join(str(tmpdir), 'output')
os.mkdir(out_dir)
self.encryption_handler.decrypt_files_to_dir([enc_files[1]], out_dir)
list_plain_dir = os.listdir(out_dir)
assert len(list_plain_dir) == 1
assert os.path.basename(enc_files[0]).replace('.aes',
'') not in list_plain_dir
assert os.path.basename(enc_files[1]).replace('.aes',
'') in list_plain_dir
def test_do_operation_launch(self, simulator_factory, operation_factory,
mocker):
# Prepare encrypted dir
op = operation_factory(test_user=self.test_user,
test_project=self.test_project)
sim_folder, sim_gid = simulator_factory(op=op)
self._do_operation_launch(op, sim_gid, mocker)
def _do_operation_launch(self, op, sim_gid, mocker, is_pse=False):
# Prepare encrypted dir
self.encryption_handler = EncryptionHandler(sim_gid)
job_encrypted_inputs = HPCSchedulerClient()._prepare_input(op, sim_gid)
self.encryption_handler.encrypt_inputs(job_encrypted_inputs)
encrypted_dir = self.encryption_handler.get_encrypted_dir()
mocker.patch('tvb.core.operation_hpc_launcher._request_passfile',
_request_passfile_dummy)
mocker.patch(
'tvb.core.operation_hpc_launcher._update_operation_status',
_update_operation_status)
# Call do_operation_launch similarly to CSCS env
plain_dir = self.files_helper.get_project_folder(
self.test_project, 'plain')
do_operation_launch(sim_gid.hex, 1000, is_pse, '', op.id, plain_dir)
assert len(os.listdir(encrypted_dir)) == 7
output_path = os.path.join(encrypted_dir,
HPCSimulatorAdapter.OUTPUT_FOLDER)
assert os.path.exists(output_path)
expected_files = 2
if is_pse:
expected_files = 3
assert len(os.listdir(output_path)) == expected_files
return output_path
def test_do_operation_launch_pse(self, simulator_factory,
operation_factory, mocker):
op = operation_factory(test_user=self.test_user,
test_project=self.test_project)
sim_folder, sim_gid = simulator_factory(op=op)
self._do_operation_launch(op, sim_gid, mocker, is_pse=True)
def test_prepare_inputs(self, operation_factory, simulator_factory):
op = operation_factory(test_user=self.test_user,
test_project=self.test_project)
sim_folder, sim_gid = simulator_factory(op=op)
hpc_client = HPCSchedulerClient()
input_files = hpc_client._prepare_input(op, sim_gid)
assert len(input_files) == 6
def test_prepare_inputs_with_surface(self, operation_factory,
simulator_factory):
op = operation_factory(test_user=self.test_user,
test_project=self.test_project)
sim_folder, sim_gid = simulator_factory(op=op, with_surface=True)
hpc_client = HPCSchedulerClient()
input_files = hpc_client._prepare_input(op, sim_gid)
assert len(input_files) == 9
def test_prepare_inputs_with_eeg_monitor(self, operation_factory,
simulator_factory,
surface_index_factory,
sensors_index_factory,
region_mapping_index_factory,
connectivity_index_factory):
surface_idx, surface = surface_index_factory(cortical=True)
sensors_idx, sensors = sensors_index_factory()
proj = ProjectionSurfaceEEG(sensors=sensors,
sources=surface,
projection_data=numpy.ones(3))
op = operation_factory()
storage_path = FilesHelper().get_project_folder(op.project, str(op.id))
prj_db_db = h5.store_complete(proj, storage_path)
prj_db_db.fk_from_operation = op.id
dao.store_entity(prj_db_db)
connectivity = connectivity_index_factory(76, op)
rm_index = region_mapping_index_factory(conn_gid=connectivity.gid,
surface_gid=surface_idx.gid)
eeg_monitor = EEGViewModel(projection=proj.gid, sensors=sensors.gid)
eeg_monitor.region_mapping = rm_index.gid
sim_folder, sim_gid = simulator_factory(op=op,
monitor=eeg_monitor,
conn_gid=connectivity.gid)
hpc_client = HPCSchedulerClient()
input_files = hpc_client._prepare_input(op, sim_gid)
assert len(input_files) == 11
def test_stage_out_to_operation_folder(self, mocker, operation_factory,
simulator_factory,
pse_burst_configuration_factory):
burst = pse_burst_configuration_factory(self.test_project)
op = operation_factory(test_user=self.test_user,
test_project=self.test_project)
op.fk_operation_group = burst.fk_operation_group
dao.store_entity(op)
sim_folder, sim_gid = simulator_factory(op=op)
burst.simulator_gid = sim_gid.hex
dao.store_entity(burst)
output_path = self._do_operation_launch(op,
sim_gid,
mocker,
is_pse=True)
def _stage_out_dummy(dir, sim_gid):
return [
os.path.join(output_path, enc_file)
for enc_file in os.listdir(output_path)
]
mocker.patch.object(HPCSchedulerClient, '_stage_out_results',
_stage_out_dummy)
sim_results_files, metric_op, metric_file = HPCSchedulerClient.stage_out_to_operation_folder(
None, op, sim_gid)
assert op.id != metric_op.id
assert os.path.exists(metric_file)
assert len(sim_results_files) == 1
assert os.path.exists(sim_results_files[0])
def teardown_method(self):
encrypted_dir = self.encryption_handler.get_encrypted_dir()
if os.path.isdir(encrypted_dir):
shutil.rmtree(encrypted_dir)
passfile = self.encryption_handler.get_password_file()
if os.path.exists(passfile):
os.remove(passfile)
self.files_helper.remove_project_structure(self.test_project.name)
self.clean_database()
class SimulatorService(object):
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.burst_service = BurstService()
self.operation_service = OperationService()
self.files_helper = FilesHelper()
def _reset_model(self, session_stored_simulator):
session_stored_simulator.model = type(session_stored_simulator.model)()
vi_indexes = MonitorForm.determine_indexes_for_chosen_vars_of_interest(
session_stored_simulator)
vi_indexes = numpy.array(list(vi_indexes.values()))
for monitor in session_stored_simulator.monitors:
monitor.variables_of_interest = vi_indexes
def reset_at_connectivity_change(self, is_simulator_copy, form,
session_stored_simulator):
"""
In case the user copies a simulation and changes the Connectivity, we want to reset the Model and Noise
parameters because they might not fit to the new Connectivity's nr of regions.
"""
if is_simulator_copy and form.connectivity.value != session_stored_simulator.connectivity:
self._reset_model(session_stored_simulator)
if issubclass(type(session_stored_simulator.integrator),
IntegratorStochastic):
session_stored_simulator.integrator.noise = type(
session_stored_simulator.integrator.noise)()
def reset_at_surface_change(self, is_simulator_copy, form,
session_stored_simulator):
"""
In case the user copies a surface-simulation and changes the Surface, we want to reset the Model
parameters because they might not fit to the new Surface's nr of vertices.
"""
if is_simulator_copy and (
session_stored_simulator.surface is None and form.surface.value
or session_stored_simulator.surface and form.surface.value !=
session_stored_simulator.surface.surface_gid):
self._reset_model(session_stored_simulator)
@staticmethod
def _set_simulator_range_parameter(simulator, range_parameter_name,
range_parameter_value):
range_param_name_list = range_parameter_name.split('.')
current_attr = simulator
for param_name in range_param_name_list[:len(range_param_name_list) -
1]:
current_attr = getattr(current_attr, param_name)
setattr(current_attr, range_param_name_list[-1], range_parameter_value)
def async_launch_and_prepare_simulation(self, burst_config, user, project,
simulator_algo,
session_stored_simulator):
try:
operation = self.operation_service.prepare_operation(
user.id, project.id, simulator_algo,
session_stored_simulator.gid)
ga = self.operation_service._prepare_metadata(
simulator_algo.algorithm_category, {}, None, burst_config.gid)
session_stored_simulator.generic_attributes = ga
storage_path = self.files_helper.get_project_folder(
project, str(operation.id))
h5.store_view_model(session_stored_simulator, storage_path)
burst_config = self.burst_service.update_simulation_fields(
burst_config.id, operation.id, session_stored_simulator.gid)
self.burst_service.store_burst_configuration(
burst_config, storage_path)
wf_errs = 0
try:
OperationService().launch_operation(operation.id, True)
return operation
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
if burst_config:
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
self.logger.debug(
"Finished launching workflow. The operation was launched successfully, "
+ str(wf_errs) + " had error on pre-launch steps")
except Exception as excep:
self.logger.error(excep)
if burst_config:
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
def prepare_simulation_on_server(self, user_id, project, algorithm,
zip_folder_path, simulator_file):
simulator_vm = h5.load_view_model_from_file(simulator_file)
operation = self.operation_service.prepare_operation(
user_id, project.id, algorithm, simulator_vm.gid)
storage_operation_path = self.files_helper.get_project_folder(
project, str(operation.id))
self.async_launch_simulation_on_server(operation, zip_folder_path,
storage_operation_path)
return operation
def async_launch_simulation_on_server(self, operation, zip_folder_path,
storage_operation_path):
try:
for file in os.listdir(zip_folder_path):
shutil.move(os.path.join(zip_folder_path, file),
storage_operation_path)
try:
OperationService().launch_operation(operation.id, True)
shutil.rmtree(zip_folder_path)
return operation
except Exception as excep:
self.logger.error(excep)
except Exception as excep:
self.logger.error(excep)
@staticmethod
def _set_range_param_in_dict(param_value):
if type(param_value) is numpy.ndarray:
return param_value[0]
elif isinstance(param_value, uuid.UUID):
return param_value.hex
else:
return param_value
def async_launch_and_prepare_pse(self, burst_config, user, project,
simulator_algo, range_param1,
range_param2, session_stored_simulator):
try:
algo_category = simulator_algo.algorithm_category
operation_group = burst_config.operation_group
metric_operation_group = burst_config.metric_operation_group
operations = []
range_param2_values = [None]
if range_param2:
range_param2_values = range_param2.get_range_values()
first_simulator = None
ga = self.operation_service._prepare_metadata(
simulator_algo.algorithm_category, {}, operation_group,
burst_config.gid)
session_stored_simulator.generic_attributes = ga
for param1_value in range_param1.get_range_values():
for param2_value in range_param2_values:
# Copy, but generate a new GUID for every Simulator in PSE
simulator = copy.deepcopy(session_stored_simulator)
simulator.gid = uuid.uuid4()
self._set_simulator_range_parameter(
simulator, range_param1.name, param1_value)
ranges = {
range_param1.name:
self._set_range_param_in_dict(param1_value)
}
if param2_value is not None:
self._set_simulator_range_parameter(
simulator, range_param2.name, param2_value)
ranges[
range_param2.name] = self._set_range_param_in_dict(
param2_value)
ranges = json.dumps(ranges)
operation = self.operation_service.prepare_operation(
user.id, project.id, simulator_algo, simulator.gid,
operation_group, ranges)
storage_path = self.files_helper.get_project_folder(
project, str(operation.id))
h5.store_view_model(simulator, storage_path)
operations.append(operation)
if first_simulator is None:
first_simulator = simulator
first_operation = operations[0]
storage_path = self.files_helper.get_project_folder(
project, str(first_operation.id))
burst_config = self.burst_service.update_simulation_fields(
burst_config.id, first_operation.id, first_simulator.gid)
self.burst_service.store_burst_configuration(
burst_config, storage_path)
datatype_group = DataTypeGroup(
operation_group,
operation_id=first_operation.id,
fk_parent_burst=burst_config.gid,
state=algo_category.defaultdatastate)
dao.store_entity(datatype_group)
metrics_datatype_group = DataTypeGroup(
metric_operation_group, fk_parent_burst=burst_config.gid)
dao.store_entity(metrics_datatype_group)
wf_errs = 0
for operation in operations:
try:
OperationService().launch_operation(operation.id, True)
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
self.logger.debug("Finished launching workflows. " +
str(len(operations) - wf_errs) +
" were launched successfully, " + str(wf_errs) +
" had error on pre-launch steps")
return first_operation
except Exception as excep:
self.logger.error(excep)
self.burst_service.mark_burst_finished(burst_config,
error_message=str(excep))
def load_from_zip(self, zip_file, project):
import_service = ImportService()
simulator_folder = import_service.import_simulator_configuration_zip(
zip_file)
simulator_h5_filename = DirLoader(
simulator_folder,
None).find_file_for_has_traits_type(SimulatorAdapterModel)
simulator_h5_filepath = os.path.join(simulator_folder,
simulator_h5_filename)
simulator = h5.load_view_model_from_file(simulator_h5_filepath)
burst_config = self.burst_service.load_burst_configuration_from_folder(
simulator_folder, project)
return simulator, burst_config
class ProjectService:
"""
Services layer for Project entities.
"""
def __init__(self):
self.logger = get_logger(__name__)
self.structure_helper = FilesHelper()
def store_project(self, current_user, is_create, selected_id, **data):
"""
We want to create/update a project entity.
"""
# Validate Unique Name
new_name = data["name"]
if len(new_name) < 1:
raise ProjectServiceException("Invalid project name!")
projects_no = dao.count_projects_for_name(new_name, selected_id)
if projects_no > 0:
err = {'name': 'Please choose another name, this one is used!'}
raise formencode.Invalid("Duplicate Name Error", {}, None, error_dict=err)
started_operations = dao.get_operation_numbers(selected_id)[1]
if started_operations > 0:
raise ProjectServiceException("A project can not be renamed while operations are still running!")
if is_create:
current_proj = model.Project(new_name, current_user.id, data["description"])
self.structure_helper.get_project_folder(current_proj)
else:
try:
current_proj = dao.get_project_by_id(selected_id)
except Exception as excep:
self.logger.exception("An error has occurred!")
raise ProjectServiceException(str(excep))
if current_proj.name != new_name:
self.structure_helper.rename_project_structure(current_proj.name, new_name)
current_proj.name = new_name
current_proj.description = data["description"]
# Commit to make sure we have a valid ID
current_proj.refresh_update_date()
self.structure_helper.write_project_metadata(current_proj)
current_proj = dao.store_entity(current_proj)
# Retrieve, to initialize lazy attributes
current_proj = dao.get_project_by_id(current_proj.id)
# Update share settings on current Project entity
visited_pages = []
prj_admin = current_proj.administrator.username
if 'visited_pages' in data and data['visited_pages']:
visited_pages = data['visited_pages'].split(',')
for page in visited_pages:
members = UserService.retrieve_all_users(prj_admin, int(page))[0]
members = [m.id for m in members]
dao.delete_members_for_project(current_proj.id, members)
selected_user_ids = data["users"]
dao.add_members_to_project(current_proj.id, selected_user_ids)
# Finish operation
self.logger.debug("Edit/Save OK for project:" + str(current_proj.id) + ' by user:'******'-'
result["count"] = one_op[2]
result["gid"] = one_op[13]
if one_op[3] is not None and one_op[3]:
try:
operation_group = dao.get_generic_entity(model.OperationGroup, one_op[3])[0]
result["group"] = operation_group.name
result["group"] = result["group"].replace("_", " ")
result["operation_group_id"] = operation_group.id
datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3])
result["datatype_group_gid"] = datatype_group.gid
result["gid"] = operation_group.gid
## Filter only viewers for current DataTypeGroup entity:
result["view_groups"] = FlowService().get_visualizers_for_group(datatype_group.gid)
except Exception:
self.logger.exception("We will ignore group on entity:" + str(one_op))
result["datatype_group_gid"] = None
else:
result['group'] = None
result['datatype_group_gid'] = None
result["algorithm"] = dao.get_algorithm_by_id(one_op[4])
result["user"] = dao.get_user_by_id(one_op[5])
if type(one_op[6]) in (str, unicode):
result["create"] = string2date(str(one_op[6]))
else:
result["create"] = one_op[6]
if type(one_op[7]) in (str, unicode):
result["start"] = string2date(str(one_op[7]))
else:
result["start"] = one_op[7]
if type(one_op[8]) in (str, unicode):
result["complete"] = string2date(str(one_op[8]))
else:
result["complete"] = one_op[8]
if result["complete"] is not None and result["start"] is not None:
result["duration"] = format_timedelta(result["complete"] - result["start"])
result["status"] = one_op[9]
result["additional"] = one_op[10]
result["visible"] = True if one_op[11] > 0 else False
result['operation_tag'] = one_op[12]
result['figures'] = None
if not result['group']:
datatype_results = dao.get_results_for_operation(result['id'])
result['results'] = []
for dt in datatype_results:
dt_loaded = ABCAdapter.load_entity_by_gid(dt.gid)
if dt_loaded:
result['results'].append(dt_loaded)
else:
self.logger.warn("Could not retrieve datatype %s" % str(dt))
operation_figures = dao.get_figures_for_operation(result['id'])
# Compute the full path to the figure / image on disk
for figure in operation_figures:
figures_folder = self.structure_helper.get_images_folder(figure.project.name)
figure_full_path = os.path.join(figures_folder, figure.file_path)
# Compute the path available from browser
figure.figure_path = utils.path2url_part(figure_full_path)
result['figures'] = operation_figures
else:
result['results'] = None
operations.append(result)
except Exception:
## We got an exception when processing one Operation Row. We will continue with the rest of the rows.
self.logger.exception("Could not prepare operation for display:" + str(one_op))
return selected_project, total_ops_nr, operations, pages_no
def retrieve_projects_for_user(self, user_id, current_page=1):
"""
Return a list with all Projects visible for current user.
"""
start_idx = PROJECTS_PAGE_SIZE * (current_page - 1)
total = dao.get_projects_for_user(user_id, is_count=True)
available_projects = dao.get_projects_for_user(user_id, start_idx, PROJECTS_PAGE_SIZE)
pages_no = total // PROJECTS_PAGE_SIZE + (1 if total % PROJECTS_PAGE_SIZE else 0)
for prj in available_projects:
fns, sta, err, canceled, pending = dao.get_operation_numbers(prj.id)
prj.operations_finished = fns
prj.operations_started = sta
prj.operations_error = err
prj.operations_canceled = canceled
prj.operations_pending = pending
prj.disk_size = dao.get_project_disk_size(prj.id)
prj.disk_size_human = format_bytes_human(prj.disk_size)
self.logger.debug("Displaying " + str(len(available_projects)) + " projects in UI for user " + str(user_id))
return available_projects, pages_no
@staticmethod
def get_linkable_projects_for_user(user_id, data_id):
"""
Find projects with are visible for current user, and in which current datatype hasn't been linked yet.
"""
return dao.get_linkable_projects_for_user(user_id, data_id)
@transactional
def remove_project(self, project_id):
"""
Remove Project from DB and File Storage.
"""
try:
project2delete = dao.get_project_by_id(project_id)
self.logger.debug("Deleting project: id=" + str(project_id) + ' name=' + project2delete.name)
project_bursts = dao.get_bursts_for_project(project_id)
for burst in project_bursts:
dao.remove_entity(burst.__class__, burst.id)
project_datatypes = dao.get_datatypes_in_project(project_id)
for one_data in project_datatypes:
self.remove_datatype(project_id, one_data.gid, True)
links = dao.get_links_for_project(project_id)
for one_link in links:
dao.remove_entity(model.Links, one_link.id)
self.structure_helper.remove_project_structure(project2delete.name)
dao.delete_project(project_id)
self.logger.debug("Deleted project: id=" + str(project_id) + ' name=' + project2delete.name)
except RemoveDataTypeException as excep:
self.logger.exception("Could not execute operation Node Remove!")
raise ProjectServiceException(str(excep))
except FileStructureException as excep:
self.logger.exception("Could not delete because of rights!")
raise ProjectServiceException(str(excep))
except Exception as excep:
self.logger.exception(str(excep))
raise ProjectServiceException(str(excep))
# ----------------- Methods for populating Data-Structure Page ---------------
@staticmethod
def get_datatype_in_group(group):
"""
Return all dataTypes that are the result of the same DTgroup.
"""
return dao.get_datatype_in_group(datatype_group_id=group)
@staticmethod
def get_datatypes_from_datatype_group(datatype_group_id):
"""
Retrieve all dataType which are part from the given dataType group.
"""
return dao.get_datatypes_from_datatype_group(datatype_group_id)
@staticmethod
def load_operation_by_gid(operation_gid):
""" Retrieve loaded Operation from DB"""
return dao.get_operation_by_gid(operation_gid)
@staticmethod
def get_operation_group_by_id(operation_group_id):
""" Loads OperationGroup from DB"""
return dao.get_operationgroup_by_id(operation_group_id)
@staticmethod
def get_operation_group_by_gid(operation_group_gid):
""" Loads OperationGroup from DB"""
return dao.get_operationgroup_by_gid(operation_group_gid)
@staticmethod
def get_operations_in_group(operation_group):
""" Return all the operations from an operation group. """
return dao.get_operations_in_group(operation_group.id)
@staticmethod
def is_upload_operation(operation_gid):
""" Returns True only if the operation with the given GID is an upload operation. """
return dao.is_upload_operation(operation_gid)
@staticmethod
def get_all_operations_for_uploaders(project_id):
""" Returns all finished upload operations. """
return dao.get_all_operations_for_uploaders(project_id)
def set_operation_and_group_visibility(self, entity_gid, is_visible, is_operation_group=False):
"""
Sets the operation visibility.
If 'is_operation_group' is True than this method will change the visibility for all
the operation from the OperationGroup with the GID field equal to 'entity_gid'.
"""
def set_visibility(op):
# workaround:
# 'reload' the operation so that it has the project property set.
# get_operations_in_group does not eager load it and now we're out of a sqlalchemy session
# write_operation_metadata requires that property
op = dao.get_operation_by_id(op.id)
# end hack
op.visible = is_visible
self.structure_helper.write_operation_metadata(op)
dao.store_entity(op)
def set_group_descendants_visibility(operation_group_id):
ops_in_group = dao.get_operations_in_group(operation_group_id)
for group_op in ops_in_group:
set_visibility(group_op)
if is_operation_group:
op_group_id = dao.get_operationgroup_by_gid(entity_gid).id
set_group_descendants_visibility(op_group_id)
else:
operation = dao.get_operation_by_gid(entity_gid)
# we assure that if the operation belongs to a group than the visibility will be changed for the entire group
if operation.fk_operation_group is not None:
set_group_descendants_visibility(operation.fk_operation_group)
else:
set_visibility(operation)
def get_operation_details(self, operation_gid, is_group):
"""
:returns: an entity OperationOverlayDetails filled with all information for current operation details.
"""
if is_group:
operation_group = self.get_operation_group_by_gid(operation_gid)
operation = dao.get_operations_in_group(operation_group.id, False, True)
## Reload, to make sure all attributes lazy are populated as well.
operation = dao.get_operation_by_gid(operation.gid)
no_of_op_in_group = dao.get_operations_in_group(operation_group.id, is_count=True)
datatype_group = self.get_datatypegroup_by_op_group_id(operation_group.id)
count_result = dao.count_datatypes_in_group(datatype_group.id)
else:
operation = dao.get_operation_by_gid(operation_gid)
if operation is None:
return None
no_of_op_in_group = 1
count_result = dao.count_resulted_datatypes(operation.id)
username = dao.get_user_by_id(operation.fk_launched_by).username
burst = dao.get_burst_for_operation_id(operation.id)
datatypes_param, all_special_params = self._review_operation_inputs(operation.gid)
op_pid = dao.get_operation_process_for_operation(operation.id)
op_details = OperationOverlayDetails(operation, username, len(datatypes_param),
count_result, burst, no_of_op_in_group, op_pid)
## Add all parameter which are set differently by the user on this Operation.
if all_special_params is not None:
op_details.add_scientific_fields(all_special_params)
return op_details
@staticmethod
def get_filterable_meta():
"""
Contains all the attributes by which
the user can structure the tree of DataTypes
"""
return DataTypeMetaData.get_filterable_meta()
def get_project_structure(self, project, visibility_filter, first_level, second_level, filter_value):
"""
Find all DataTypes (including the linked ones and the groups) relevant for the current project.
In case of a problem, will return an empty list.
"""
metadata_list = []
dt_list = dao.get_data_in_project(project.id, visibility_filter, filter_value)
for dt in dt_list:
# Prepare the DT results from DB, for usage in controller, by converting into DataTypeMetaData objects
data = {}
is_group = False
group_op = None
dt_entity = dao.get_datatype_by_gid(dt.gid)
if dt_entity is None:
self.logger.warning("Ignored entity (possibly removed DT class)" + str(dt))
continue
## Filter by dt.type, otherwise Links to individual DT inside a group will be mistaken
if dt.type == "DataTypeGroup" and dt.parent_operation.operation_group is not None:
is_group = True
group_op = dt.parent_operation.operation_group
# All these fields are necessary here for dynamic Tree levels.
data[DataTypeMetaData.KEY_DATATYPE_ID] = dt.id
data[DataTypeMetaData.KEY_GID] = dt.gid
data[DataTypeMetaData.KEY_NODE_TYPE] = dt.type
data[DataTypeMetaData.KEY_STATE] = dt.state
data[DataTypeMetaData.KEY_SUBJECT] = str(dt.subject)
data[DataTypeMetaData.KEY_TITLE] = dt_entity.display_name
data[DataTypeMetaData.KEY_RELEVANCY] = dt.visible
data[DataTypeMetaData.KEY_LINK] = dt.parent_operation.fk_launched_in != project.id
data[DataTypeMetaData.KEY_TAG_1] = dt.user_tag_1 if dt.user_tag_1 else ''
data[DataTypeMetaData.KEY_TAG_2] = dt.user_tag_2 if dt.user_tag_2 else ''
data[DataTypeMetaData.KEY_TAG_3] = dt.user_tag_3 if dt.user_tag_3 else ''
data[DataTypeMetaData.KEY_TAG_4] = dt.user_tag_4 if dt.user_tag_4 else ''
data[DataTypeMetaData.KEY_TAG_5] = dt.user_tag_5 if dt.user_tag_5 else ''
# Operation related fields:
operation_name = CommonDetails.compute_operation_name(
dt.parent_operation.algorithm.algorithm_category.displayname,
dt.parent_operation.algorithm.displayname)
data[DataTypeMetaData.KEY_OPERATION_TYPE] = operation_name
data[DataTypeMetaData.KEY_OPERATION_ALGORITHM] = dt.parent_operation.algorithm.displayname
data[DataTypeMetaData.KEY_AUTHOR] = dt.parent_operation.user.username
data[DataTypeMetaData.KEY_OPERATION_TAG] = group_op.name if is_group else dt.parent_operation.user_group
data[DataTypeMetaData.KEY_OP_GROUP_ID] = group_op.id if is_group else None
completion_date = dt.parent_operation.completion_date
string_year = completion_date.strftime(MONTH_YEAR_FORMAT) if completion_date is not None else ""
string_month = completion_date.strftime(DAY_MONTH_YEAR_FORMAT) if completion_date is not None else ""
data[DataTypeMetaData.KEY_DATE] = date2string(completion_date) if (completion_date is not None) else ''
data[DataTypeMetaData.KEY_CREATE_DATA_MONTH] = string_year
data[DataTypeMetaData.KEY_CREATE_DATA_DAY] = string_month
data[DataTypeMetaData.KEY_BURST] = dt._parent_burst.name if dt._parent_burst is not None else '-None-'
metadata_list.append(DataTypeMetaData(data, dt.invalid))
return StructureNode.metadata2tree(metadata_list, first_level, second_level, project.id, project.name)
@staticmethod
def get_datatype_details(datatype_gid):
"""
:returns: an array. First entry in array is an instance of DataTypeOverlayDetails\
The second one contains all the possible states for the specified dataType.
"""
meta_atts = DataTypeOverlayDetails()
states = DataTypeMetaData.STATES
try:
datatype_result = dao.get_datatype_details(datatype_gid)
meta_atts.fill_from_datatype(datatype_result, datatype_result._parent_burst)
return meta_atts, states, datatype_result
except Exception:
## We ignore exception here (it was logged above, and we want to return no details).
return meta_atts, states, None
def _remove_project_node_files(self, project_id, gid, skip_validation=False):
"""
Delegate removal of a node in the structure of the project.
In case of a problem will THROW StructureException.
"""
try:
project = self.find_project(project_id)
datatype = dao.get_datatype_by_gid(gid)
links = dao.get_links_for_datatype(datatype.id)
if links:
was_link = False
for link in links:
# This means it's only a link and we need to remove it
if link.fk_from_datatype == datatype.id and link.fk_to_project == project.id:
dao.remove_entity(model.Links, link.id)
was_link = True
if not was_link:
# Create a clone of the operation
new_op = model.Operation(dao.get_system_user().id,
links[0].fk_to_project,
datatype.parent_operation.fk_from_algo,
datatype.parent_operation.parameters,
datatype.parent_operation.meta_data,
datatype.parent_operation.status,
datatype.parent_operation.start_date,
datatype.parent_operation.completion_date,
datatype.parent_operation.fk_operation_group,
datatype.parent_operation.additional_info,
datatype.parent_operation.user_group,
datatype.parent_operation.range_values)
new_op = dao.store_entity(new_op)
to_project = self.find_project(links[0].fk_to_project).name
new_op_loaded = dao.get_operation_by_id(new_op.id)
self.structure_helper.write_operation_metadata(new_op_loaded)
self.structure_helper.move_datatype(datatype, to_project, str(new_op.id))
datatype.set_operation_id(new_op.id)
datatype.parent_operation = new_op
dao.store_entity(datatype)
dao.remove_entity(model.Links, links[0].id)
else:
specific_remover = get_remover(datatype.type)(datatype)
specific_remover.remove_datatype(skip_validation)
self.structure_helper.remove_datatype(datatype)
except RemoveDataTypeException:
self.logger.exception("Could not execute operation Node Remove!")
raise
except FileStructureException:
self.logger.exception("Remove operation failed")
raise StructureException("Remove operation failed for unknown reasons.Please contact system administrator.")
def remove_operation(self, operation_id):
"""
Remove a given operation
"""
operation = dao.try_get_operation_by_id(operation_id)
if operation is not None:
self.logger.debug("Deleting operation %s " % operation)
datatypes_for_op = dao.get_results_for_operation(operation_id)
for dt in reversed(datatypes_for_op):
self.remove_datatype(operation.project.id, dt.gid, False)
dao.remove_entity(model.Operation, operation.id)
self.logger.debug("Finished deleting operation %s " % operation)
else:
self.logger.warning("Attempt to delete operation with id=%s which no longer exists." % operation_id)
def remove_datatype(self, project_id, datatype_gid, skip_validation=False):
"""
Method used for removing a dataType. If the given dataType is a DatatypeGroup
or a dataType from a DataTypeGroup than this method will remove the entire group.
The operation(s) used for creating the dataType(s) will also be removed.
"""
datatype = dao.get_datatype_by_gid(datatype_gid)
if datatype is None:
self.logger.warning("Attempt to delete DT[%s] which no longer exists." % datatype_gid)
return
is_datatype_group = False
if dao.is_datatype_group(datatype_gid):
is_datatype_group = True
elif datatype.fk_datatype_group is not None:
is_datatype_group = True
datatype = dao.get_datatype_by_id(datatype.fk_datatype_group)
operations_set = [datatype.fk_from_operation]
correct = True
if is_datatype_group:
self.logger.debug("Removing datatype group %s" % datatype)
data_list = dao.get_datatypes_from_datatype_group(datatype.id)
for adata in data_list:
self._remove_project_node_files(project_id, adata.gid, skip_validation)
if adata.fk_from_operation not in operations_set:
operations_set.append(adata.fk_from_operation)
datatype_group = dao.get_datatype_group_by_gid(datatype.gid)
dao.remove_datatype(datatype_gid)
correct = correct and dao.remove_entity(model.OperationGroup, datatype_group.fk_operation_group)
else:
self.logger.debug("Removing datatype %s" % datatype)
self._remove_project_node_files(project_id, datatype.gid, skip_validation)
## Remove Operation entity in case no other DataType needs them.
project = dao.get_project_by_id(project_id)
for operation_id in operations_set:
dependent_dt = dao.get_generic_entity(model.DataType, operation_id, "fk_from_operation")
if len(dependent_dt) > 0:
### Do not remove Operation in case DataType still exist referring it.
continue
correct = correct and dao.remove_entity(model.Operation, operation_id)
## Make sure Operation folder is removed
self.structure_helper.remove_operation_data(project.name, datatype.fk_from_operation)
if not correct:
raise RemoveDataTypeException("Could not remove DataType " + str(datatype_gid))
def update_metadata(self, submit_data):
"""
Update DataType/ DataTypeGroup metadata
THROW StructureException when input data is invalid.
"""
new_data = dict()
for key in DataTypeOverlayDetails().meta_attributes_list:
if key in submit_data:
new_data[key] = submit_data[key]
if new_data[CommonDetails.CODE_OPERATION_TAG] == '':
new_data[CommonDetails.CODE_OPERATION_TAG] = None
try:
if (CommonDetails.CODE_OPERATION_GROUP_ID in new_data
and new_data[CommonDetails.CODE_OPERATION_GROUP_ID]
and new_data[CommonDetails.CODE_OPERATION_GROUP_ID] != ''):
# We need to edit a group
all_data_in_group = dao.get_datatype_in_group(operation_group_id=
new_data[CommonDetails.CODE_OPERATION_GROUP_ID])
if len(all_data_in_group) < 1:
raise StructureException("Inconsistent group, can not be updated!")
datatype_group = dao.get_generic_entity(model.DataTypeGroup, all_data_in_group[0].fk_datatype_group)[0]
all_data_in_group.append(datatype_group)
for datatype in all_data_in_group:
new_data[CommonDetails.CODE_GID] = datatype.gid
self._edit_data(datatype, new_data, True)
else:
# Get the required DataType and operation from DB to store changes that will be done in XML.
gid = new_data[CommonDetails.CODE_GID]
datatype = dao.get_datatype_by_gid(gid)
self._edit_data(datatype, new_data)
except Exception as excep:
self.logger.exception(excep)
raise StructureException(str(excep))
def _edit_data(self, datatype, new_data, from_group=False):
"""
Private method, used for editing a meta-data XML file and a DataType row
for a given custom DataType entity with new dictionary of data from UI.
"""
if isinstance(datatype, MappedType) and not os.path.exists(datatype.get_storage_file_path()):
if not datatype.invalid:
datatype.invalid = True
dao.store_entity(datatype)
return
# 1. First update Operation fields:
# Update group field if possible
new_group_name = new_data[CommonDetails.CODE_OPERATION_TAG]
empty_group_value = (new_group_name is None or new_group_name == "")
if from_group:
if empty_group_value:
raise StructureException("Empty group is not allowed!")
group = dao.get_generic_entity(model.OperationGroup, new_data[CommonDetails.CODE_OPERATION_GROUP_ID])
if group and len(group) > 0 and new_group_name != group[0].name:
group = group[0]
exists_group = dao.get_generic_entity(model.OperationGroup, new_group_name, 'name')
if exists_group:
raise StructureException("Group '" + new_group_name + "' already exists.")
group.name = new_group_name
dao.store_entity(group)
else:
operation = dao.get_operation_by_id(datatype.fk_from_operation)
operation.user_group = new_group_name
dao.store_entity(operation)
# 2. Update dateType fields:
datatype.subject = new_data[DataTypeOverlayDetails.DATA_SUBJECT]
datatype.state = new_data[DataTypeOverlayDetails.DATA_STATE]
if DataTypeOverlayDetails.DATA_TAG_1 in new_data:
datatype.user_tag_1 = new_data[DataTypeOverlayDetails.DATA_TAG_1]
if DataTypeOverlayDetails.DATA_TAG_2 in new_data:
datatype.user_tag_2 = new_data[DataTypeOverlayDetails.DATA_TAG_2]
if DataTypeOverlayDetails.DATA_TAG_3 in new_data:
datatype.user_tag_3 = new_data[DataTypeOverlayDetails.DATA_TAG_3]
if DataTypeOverlayDetails.DATA_TAG_4 in new_data:
datatype.user_tag_4 = new_data[DataTypeOverlayDetails.DATA_TAG_4]
if DataTypeOverlayDetails.DATA_TAG_5 in new_data:
datatype.user_tag_5 = new_data[DataTypeOverlayDetails.DATA_TAG_5]
datatype = dao.store_entity(datatype)
# 3. Update MetaData in H5 as well.
datatype.persist_full_metadata()
# 4. Update the group_name/user_group into the operation meta-data file
operation = dao.get_operation_by_id(datatype.fk_from_operation)
self.structure_helper.update_operation_metadata(operation.project.name, new_group_name,
str(datatype.fk_from_operation), from_group)
def get_datatype_and_datatypegroup_inputs_for_operation(self, operation_gid, selected_filter):
"""
Returns the dataTypes that are used as input parameters for the given operation.
'selected_filter' - is expected to be a visibility filter.
If any dataType is part of a dataType group then the dataType group will
be returned instead of that dataType.
"""
all_datatypes = self._review_operation_inputs(operation_gid)[0]
datatype_inputs = []
for datatype in all_datatypes:
if selected_filter.display_name == StaticFiltersFactory.RELEVANT_VIEW:
if datatype.visible:
datatype_inputs.append(datatype)
else:
datatype_inputs.append(datatype)
datatypes = []
datatype_groups = dict()
for data_type in datatype_inputs:
if data_type.fk_datatype_group is None:
datatypes.append(data_type)
elif data_type.fk_datatype_group not in datatype_groups:
dt_group = dao.get_datatype_by_id(data_type.fk_datatype_group)
datatype_groups[data_type.fk_datatype_group] = dt_group
datatypes.extend([v for _, v in six.iteritems(datatype_groups)])
return datatypes
def _review_operation_inputs(self, operation_gid):
"""
:returns: A list of DataTypes that are used as input parameters for the specified operation.
And a dictionary will all operation parameters different then the default ones.
"""
operation = dao.get_operation_by_gid(operation_gid)
parameters = json.loads(operation.parameters)
try:
adapter = ABCAdapter.build_adapter(operation.algorithm)
return adapter.review_operation_inputs(parameters)
except IntrospectionException:
self.logger.warning("Could not find adapter class for operation %s" % operation_gid)
inputs_datatypes = []
changed_parameters = dict(Warning="Algorithm was Removed. We can not offer more details")
for submit_param in parameters.values():
self.logger.debug("Searching DT by GID %s" % submit_param)
datatype = ABCAdapter.load_entity_by_gid(str(submit_param))
if datatype is not None:
inputs_datatypes.append(datatype)
return inputs_datatypes, changed_parameters
def get_datatypes_inputs_for_operation_group(self, group_id, selected_filter):
"""
Returns the dataType inputs for an operation group. If more dataTypes
are part of the same dataType group then only the dataType group will
be returned instead of them.
"""
operations_gids = dao.get_operations_in_group(group_id, only_gids=True)
op_group_inputs = dict()
for gid in operations_gids:
op_inputs = self.get_datatype_and_datatypegroup_inputs_for_operation(gid[0], selected_filter)
for datatype in op_inputs:
op_group_inputs[datatype.id] = datatype
return op_group_inputs.values()
@staticmethod
def get_results_for_operation(operation_id, selected_filter=None):
"""
Retrieve the DataTypes entities resulted after the execution of the given operation.
"""
return dao.get_results_for_operation(operation_id, selected_filter)
@staticmethod
def get_operations_for_datatype_group(datatype_group_id, visibility_filter, only_in_groups=False):
"""
Returns all the operations which uses as an input parameter a dataType from the given DataTypeGroup.
visibility_filter - is a filter used for retrieving all the operations or only the relevant ones.
If only_in_groups is True than this method will return only the operations that are
part from an operation group, otherwise it will return only the operations that
are NOT part of an operation group.
"""
if visibility_filter.display_name != StaticFiltersFactory.RELEVANT_VIEW:
return dao.get_operations_for_datatype_group(datatype_group_id, only_relevant=False,
only_in_groups=only_in_groups)
return dao.get_operations_for_datatype_group(datatype_group_id, only_in_groups=only_in_groups)
@staticmethod
def get_operations_for_datatype(datatype_gid, visibility_filter, only_in_groups=False):
"""
Returns all the operations which uses as an input parameter the dataType with the specified GID.
If only_in_groups is True than this method will return only the operations that are part
from an operation group, otherwise it will return only the operations that are NOT part of an operation group.
"""
if visibility_filter.display_name != StaticFiltersFactory.RELEVANT_VIEW:
return dao.get_operations_for_datatype(datatype_gid, only_relevant=False, only_in_groups=only_in_groups)
return dao.get_operations_for_datatype(datatype_gid, only_in_groups=only_in_groups)
@staticmethod
def get_datatype_by_id(datatype_id):
"""Retrieve a DataType DB reference by its id."""
return dao.get_datatype_by_id(datatype_id)
@staticmethod
def get_datatypegroup_by_gid(datatypegroup_gid):
""" Returns the DataTypeGroup with the specified gid. """
return dao.get_datatype_group_by_gid(datatypegroup_gid)
@staticmethod
def count_datatypes_generated_from(datatype_gid):
"""
A list with all the datatypes resulted from operations that had as
input the datatype given by 'datatype_gid'.
"""
return dao.count_datatypes_generated_from(datatype_gid)
@staticmethod
def get_datatypegroup_by_op_group_id(operation_group_id):
""" Returns the DataTypeGroup with the specified id. """
return dao.get_datatypegroup_by_op_group_id(operation_group_id)
@staticmethod
def set_datatype_visibility(datatype_gid, is_visible):
"""
Sets the dataType visibility. If the given dataType is a dataType group or it is part of a
dataType group than this method will set the visibility for each dataType from this group.
"""
def set_visibility(dt):
""" set visibility flag, persist in db and h5"""
dt.visible = is_visible
dt = dao.store_entity(dt)
dt.persist_full_metadata()
def set_group_descendants_visibility(datatype_group_id):
datatypes_in_group = dao.get_datatypes_from_datatype_group(datatype_group_id)
for group_dt in datatypes_in_group:
set_visibility(group_dt)
datatype = dao.get_datatype_by_gid(datatype_gid)
if isinstance(datatype, DataTypeGroup): # datatype is a group
set_group_descendants_visibility(datatype.id)
elif datatype.fk_datatype_group is not None: # datatype is member of a group
set_group_descendants_visibility(datatype.fk_datatype_group)
# the datatype to be updated is the parent datatype group
datatype = dao.get_datatype_by_id(datatype.fk_datatype_group)
# update the datatype or datatype group.
set_visibility(datatype)
@staticmethod
def is_datatype_group(datatype_gid):
""" Used to check if the dataType with the specified GID is a DataTypeGroup. """
return dao.is_datatype_group(datatype_gid)
