def run(self):
"""
Get the required data from the operation queue and launch the operation.
"""
operation_id = self.operation_id
run_params = [TvbProfile.current.PYTHON_INTERPRETER_PATH, '-m', 'tvb.core.operation_async_launcher',
str(operation_id), TvbProfile.CURRENT_PROFILE_NAME]
current_operation = dao.get_operation_by_id(operation_id)
storage_interface = StorageInterface()
project_folder = storage_interface.get_project_folder(current_operation.project.name)
in_usage = storage_interface.is_in_usage(project_folder)
storage_interface.inc_running_op_count(project_folder)
if not in_usage:
storage_interface.sync_folders(project_folder)
# In the exceptional case where the user pressed stop while the Thread startup is done,
# We should no longer launch the operation.
if self.stopped() is False:
env = os.environ.copy()
env['PYTHONPATH'] = os.pathsep.join(sys.path)
# anything that was already in $PYTHONPATH should have been reproduced in sys.path
launched_process = Popen(run_params, stdout=PIPE, stderr=PIPE, env=env)
LOGGER.debug("Storing pid=%s for operation id=%s launched on local machine." % (operation_id,
launched_process.pid))
op_ident = OperationProcessIdentifier(operation_id, pid=launched_process.pid)
dao.store_entity(op_ident)
if self.stopped():
# In the exceptional case where the user pressed stop while the Thread startup is done.
# and stop_operation is concurrently asking about OperationProcessIdentity.
self.stop_pid(launched_process.pid)
subprocess_result = launched_process.communicate()
LOGGER.info("Finished with launch of operation %s" % operation_id)
returned = launched_process.wait()
LOGGER.info("Return code: {}. Stopped: {}".format(returned, self.stopped()))
LOGGER.info("Thread: {}".format(self))
if returned != 0 and not self.stopped():
# Process did not end as expected. (e.g. Segmentation fault)
burst_service = BurstService()
operation = dao.get_operation_by_id(self.operation_id)
LOGGER.error("Operation suffered fatal failure! Exit code: %s Exit message: %s" % (returned,
subprocess_result))
burst_service.persist_operation_state(operation, STATUS_ERROR,
"Operation failed unexpectedly! Please check the log files.")
del launched_process
storage_interface.check_and_delete(project_folder)
# Give back empty spot now that you finished your operation
CURRENT_ACTIVE_THREADS.remove(self)
LOCKS_QUEUE.put(1)
class SimulatorService(object):
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.burst_service = BurstService()
self.operation_service = OperationService()
self.algorithm_service = AlgorithmService()
self.storage_interface = StorageInterface()
@staticmethod
def _reset_model(session_stored_simulator):
session_stored_simulator.model = type(session_stored_simulator.model)()
vi_indexes = session_stored_simulator.determine_indexes_for_chosen_vars_of_interest(
)
vi_indexes = numpy.array(list(vi_indexes.values()))
for monitor in session_stored_simulator.monitors:
monitor.variables_of_interest = vi_indexes
def reset_at_connectivity_change(self, is_simulator_copy, form,
session_stored_simulator):
"""
In case the user copies a simulation and changes the Connectivity, we want to reset the Model and Noise
parameters because they might not fit to the new Connectivity's nr of regions.
"""
if is_simulator_copy and form.connectivity.value != session_stored_simulator.connectivity:
self._reset_model(session_stored_simulator)
if issubclass(type(session_stored_simulator.integrator),
IntegratorStochastic):
session_stored_simulator.integrator.noise = type(
session_stored_simulator.integrator.noise)()
def reset_at_surface_change(self, is_simulator_copy, form,
session_stored_simulator):
"""
In case the user copies a surface-simulation and changes the Surface, we want to reset the Model
parameters because they might not fit to the new Surface's nr of vertices.
"""
if is_simulator_copy and (
session_stored_simulator.surface is None and form.surface.value
or session_stored_simulator.surface and form.surface.value !=
session_stored_simulator.surface.surface_gid):
self._reset_model(session_stored_simulator)
@staticmethod
def _set_simulator_range_parameter(simulator, range_parameter_name,
range_parameter_value):
range_param_name_list = range_parameter_name.split('.')
current_attr = simulator
for param_name in range_param_name_list[:len(range_param_name_list) -
1]:
current_attr = getattr(current_attr, param_name)
setattr(current_attr, range_param_name_list[-1], range_parameter_value)
def async_launch_and_prepare_simulation(self, burst_config, user, project,
simulator_algo, simulator):
try:
operation = self.operation_service.prepare_operation(
user.id,
project,
simulator_algo,
view_model=simulator,
burst_gid=burst_config.gid,
op_group_id=burst_config.fk_operation_group)
burst_config = self.burst_service.update_simulation_fields(
burst_config, operation.id, simulator.gid)
storage_path = self.storage_interface.get_project_folder(
project.name, str(operation.id))
self.burst_service.store_burst_configuration(
burst_config, storage_path)
wf_errs = 0
try:
OperationService().launch_operation(operation.id, True)
return operation
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
if burst_config:
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
self.logger.debug(
"Finished launching workflow. The operation was launched successfully, "
+ str(wf_errs) + " had error on pre-launch steps")
except Exception as excep:
self.logger.error(excep)
if burst_config:
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
def prepare_simulation_on_server(self, user_id, project, algorithm,
zip_folder_path, simulator_file):
simulator_vm = h5.load_view_model_from_file(simulator_file)
operation = self.operation_service.prepare_operation(
user_id, project, algorithm, view_model=simulator_vm)
self.async_launch_simulation_on_server(operation, zip_folder_path)
return operation
def async_launch_simulation_on_server(self, operation, zip_folder_path):
try:
OperationService().launch_operation(operation.id, True)
return operation
except Exception as excep:
self.logger.error(excep)
finally:
shutil.rmtree(zip_folder_path)
@staticmethod
def _set_range_param_in_dict(param_value):
if type(param_value) is numpy.ndarray:
return param_value[0]
elif isinstance(param_value, uuid.UUID):
return param_value.hex
else:
return param_value
def async_launch_and_prepare_pse(self, burst_config, user, project,
simulator_algo, range_param1,
range_param2, session_stored_simulator):
try:
algo_category = simulator_algo.algorithm_category
operation_group = burst_config.operation_group
metric_operation_group = burst_config.metric_operation_group
range_param2_values = [None]
if range_param2:
range_param2_values = range_param2.get_range_values()
GROUP_BURST_PENDING[burst_config.id] = True
operations, pse_canceled = self._prepare_operations(
algo_category, burst_config, metric_operation_group,
operation_group, project, range_param1, range_param2,
range_param2_values, session_stored_simulator, simulator_algo,
user)
GROUP_BURST_PENDING[burst_config.id] = False
if pse_canceled:
return
wf_errs = self._launch_operations(operations, burst_config)
self.logger.debug("Finished launching workflows. " +
str(len(operations) - wf_errs) +
" were launched successfully, " + str(wf_errs) +
" had error on pre-launch steps")
return operations[0] if len(operations) > 0 else None
except Exception as excep:
self.logger.error(excep)
self.burst_service.mark_burst_finished(burst_config,
error_message=str(excep))
def _launch_operations(self, operations, burst_config):
wf_errs = 0
for operation in operations:
try:
burst_config = dao.get_burst_by_id(burst_config.id)
if burst_config is None or burst_config.status in [
BurstConfiguration.BURST_CANCELED,
BurstConfiguration.BURST_ERROR
]:
self.logger.debug(
"Preparing operations cannot continue. Burst config {}"
.format(burst_config))
return
OperationService().launch_operation(operation.id, True)
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
return wf_errs
def _prepare_operations(self, algo_category, burst_config,
metric_operation_group, operation_group, project,
range_param1, range_param2, range_param2_values,
session_stored_simulator, simulator_algo, user):
first_simulator = None
pse_canceled = False
operations = []
for param1_value in range_param1.get_range_values():
for param2_value in range_param2_values:
burst_config = dao.get_burst_by_id(burst_config.id)
if burst_config is None:
self.logger.debug("Burst config was deleted")
pse_canceled = True
break
if burst_config.status in [
BurstConfiguration.BURST_CANCELED,
BurstConfiguration.BURST_ERROR
]:
self.logger.debug(
"Current burst status is {}. Preparing operations cannot continue."
.format(burst_config.status))
pse_canceled = True
break
# Copy, but generate a new GUID for every Simulator in PSE
simulator = copy.deepcopy(session_stored_simulator)
simulator.gid = uuid.uuid4()
self._set_simulator_range_parameter(simulator,
range_param1.name,
param1_value)
ranges = {
range_param1.name:
self._set_range_param_in_dict(param1_value)
}
if param2_value is not None:
self._set_simulator_range_parameter(
simulator, range_param2.name, param2_value)
ranges[range_param2.name] = self._set_range_param_in_dict(
param2_value)
ranges = json.dumps(ranges)
operation = self.operation_service.prepare_operation(
user.id,
project,
simulator_algo,
view_model=simulator,
ranges=ranges,
burst_gid=burst_config.gid,
op_group_id=burst_config.fk_operation_group)
simulator.range_values = ranges
operations.append(operation)
if first_simulator is None:
first_simulator = simulator
storage_path = self.storage_interface.get_project_folder(
project.name, str(operation.id))
burst_config = self.burst_service.update_simulation_fields(
burst_config, operation.id, first_simulator.gid)
self.burst_service.store_burst_configuration(
burst_config, storage_path)
datatype_group = DataTypeGroup(
operation_group,
operation_id=operation.id,
fk_parent_burst=burst_config.gid,
state=algo_category.defaultdatastate)
dao.store_entity(datatype_group)
metrics_datatype_group = DataTypeGroup(
metric_operation_group,
fk_parent_burst=burst_config.gid,
state=algo_category.defaultdatastate)
dao.store_entity(metrics_datatype_group)
return operations, pse_canceled
@staticmethod
def compute_conn_branch_conditions(is_branch, simulator):
if not is_branch:
return None
conn = load.load_entity_by_gid(simulator.connectivity)
if conn.number_of_regions:
return FilterChain(
fields=[FilterChain.datatype + '.number_of_regions'],
operations=["=="],
values=[conn.number_of_regions])
@staticmethod
def validate_first_fragment(form, project_id, conn_idx):
conn_count = dao.count_datatypes(project_id, conn_idx)
if conn_count == 0:
form.connectivity.errors.append(
"No connectivity in the project! Simulation cannot be started without "
"a connectivity!")
def get_simulation_state_index(self, burst_config,
simulation_history_class):
parent_burst = burst_config.parent_burst_object
simulation_state_index = dao.get_generic_entity(
simulation_history_class, parent_burst.gid, "fk_parent_burst")
if simulation_state_index is None or len(simulation_state_index) < 1:
exc = BurstServiceException(
"Simulation State not found for %s, thus we are unable to branch from "
"it!" % burst_config.name)
self.logger.error(exc)
raise exc
return simulation_state_index
class ExportManager(object):
"""
This class provides basic methods for exporting data types of projects in different formats.
"""
all_exporters = {} # Dictionary containing all available exporters
export_folder = None
EXPORT_FOLDER_NAME = "EXPORT_TMP"
EXPORTED_SIMULATION_NAME = "exported_simulation"
EXPORTED_SIMULATION_DTS_DIR = "datatypes"
logger = get_logger(__name__)
def __init__(self):
# Here we register all available data type exporters
# If new exporters supported, they should be added here
self._register_exporter(TVBExporter())
self._register_exporter(TVBLinkedExporter())
self.export_folder = os.path.join(TvbProfile.current.TVB_STORAGE, self.EXPORT_FOLDER_NAME)
self.storage_interface = StorageInterface()
def _register_exporter(self, exporter):
"""
This method register into an internal format available exporters.
:param exporter: Instance of a data type exporter (extends ABCExporter)
"""
if exporter is not None:
self.all_exporters[exporter.__class__.__name__] = exporter
def get_exporters_for_data(self, data):
"""
Get available exporters for current data type.
:returns: a dictionary with the {exporter_id : label}
"""
if data is None:
raise InvalidExportDataException("Could not detect exporters for null data")
self.logger.debug("Trying to determine exporters valid for %s" % data.type)
results = {}
# No exporter for None data
if data is None:
return results
for exporterId in self.all_exporters.keys():
exporter = self.all_exporters[exporterId]
if exporter.accepts(data):
results[exporterId] = exporter.get_label()
return results
def export_data(self, data, exporter_id, project):
"""
Export provided data using given exporter
:param data: data type to be exported
:param exporter_id: identifier of the exporter to be used
:param project: project that contains data to be exported
:returns: a tuple with the following elements
1. name of the file to be shown to user
2. full path of the export file (available for download)
3. boolean which specify if file can be deleted after download
"""
if data is None:
raise InvalidExportDataException("Could not export null data. Please select data to be exported")
if exporter_id is None:
raise ExportException("Please select the exporter to be used for this operation")
if exporter_id not in self.all_exporters:
raise ExportException("Provided exporter identifier is not a valid one")
exporter = self.all_exporters[exporter_id]
if project is None:
raise ExportException("Please provide the project where data files are stored")
# Now we start the real export
if not exporter.accepts(data):
raise InvalidExportDataException("Current data can not be exported by specified exporter")
# Now compute and create folder where to store exported data
# This will imply to generate a folder which is unique for each export
data_export_folder = None
try:
data_export_folder = self.storage_interface.build_data_export_folder(data, self.export_folder)
self.logger.debug("Start export of data: %s" % data.type)
export_data = exporter.export(data, data_export_folder, project)
finally:
# In case export did not generated any file delete folder
if data_export_folder is not None and len(os.listdir(data_export_folder)) == 0:
os.rmdir(data_export_folder)
return export_data
def _export_linked_datatypes(self, project):
linked_paths = ProjectService().get_linked_datatypes_storage_path(project)
if not linked_paths:
# do not export an empty operation
return None, None
# Make an import operation which will contain links to other projects
algo = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE, TVB_IMPORTER_CLASS)
op = model_operation.Operation(None, None, project.id, algo.id)
op.project = project
op.algorithm = algo
op.id = 'links-to-external-projects'
op.start_now()
op.mark_complete(model_operation.STATUS_FINISHED)
return linked_paths, op
def export_project(self, project):
"""
Given a project root and the TVB storage_path, create a ZIP
ready for export.
:param project: project object which identifies project to be exported
"""
if project is None:
raise ExportException("Please provide project to be exported")
folders_to_exclude = self._get_op_with_errors(project.id)
linked_paths, op = self._export_linked_datatypes(project)
result_path = self.storage_interface.export_project(project, folders_to_exclude,
self.export_folder, linked_paths, op)
return result_path
@staticmethod
def _get_op_with_errors(project_id):
"""
Get the operation folders with error base name as list.
"""
operations = dao.get_operations_with_error_in_project(project_id)
op_with_errors = []
for op in operations:
op_with_errors.append(op.id)
return op_with_errors
def export_simulator_configuration(self, burst_id):
burst = dao.get_burst_by_id(burst_id)
if burst is None:
raise InvalidExportDataException("Could not find burst with ID " + str(burst_id))
op_folder = self.storage_interface.get_project_folder(burst.project.name, str(burst.fk_simulation))
tmp_export_folder = self.storage_interface.build_data_export_folder(burst, self.export_folder)
tmp_sim_folder = os.path.join(tmp_export_folder, self.EXPORTED_SIMULATION_NAME)
if not os.path.exists(tmp_sim_folder):
os.makedirs(tmp_sim_folder)
all_view_model_paths, all_datatype_paths = h5.gather_references_of_view_model(burst.simulator_gid, op_folder)
burst_path = h5.determine_filepath(burst.gid, op_folder)
all_view_model_paths.append(burst_path)
for vm_path in all_view_model_paths:
dest = os.path.join(tmp_sim_folder, os.path.basename(vm_path))
self.storage_interface.copy_file(vm_path, dest)
for dt_path in all_datatype_paths:
dest = os.path.join(tmp_sim_folder, self.EXPORTED_SIMULATION_DTS_DIR, os.path.basename(dt_path))
self.storage_interface.copy_file(dt_path, dest)
main_vm_path = h5.determine_filepath(burst.simulator_gid, tmp_sim_folder)
H5File.remove_metadata_param(main_vm_path, 'history_gid')
now = datetime.now()
date_str = now.strftime("%Y-%m-%d_%H-%M")
zip_file_name = "%s_%s.%s" % (date_str, str(burst_id), StorageInterface.ZIP_FILE_EXTENSION)
result_path = os.path.join(tmp_export_folder, zip_file_name)
self.storage_interface.write_zip_folder(result_path, tmp_sim_folder)
self.storage_interface.remove_folder(tmp_sim_folder)
return result_path
class TestSimulationResource(RestResourceTest):
def transactional_setup_method(self):
self.test_user = TestFactory.create_user('Rest_User')
self.test_project = TestFactory.create_project(
self.test_user, 'Rest_Project', users=[self.test_user.id])
self.simulation_resource = FireSimulationResource()
self.storage_interface = StorageInterface()
def test_server_fire_simulation_inexistent_gid(self, mocker):
self._mock_user(mocker)
project_gid = "inexistent-gid"
dummy_file = FileStorage(BytesIO(b"test"), 'test.zip')
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request')
request_mock.files = {
RequestFileKey.SIMULATION_FILE_KEY.value: dummy_file
}
with pytest.raises(InvalidIdentifierException):
self.simulation_resource.post(project_gid=project_gid)
def test_server_fire_simulation_no_file(self, mocker):
self._mock_user(mocker)
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request')
request_mock.files = {}
with pytest.raises(InvalidIdentifierException):
self.simulation_resource.post(project_gid='')
def test_server_fire_simulation_bad_extension(self, mocker):
self._mock_user(mocker)
dummy_file = FileStorage(BytesIO(b"test"), 'test.txt')
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request')
request_mock.files = {
RequestFileKey.SIMULATION_FILE_KEY.value: dummy_file
}
with pytest.raises(InvalidIdentifierException):
self.simulation_resource.post(project_gid='')
def test_server_fire_simulation(self, mocker, connectivity_factory):
self._mock_user(mocker)
input_folder = self.storage_interface.get_project_folder(
self.test_project.name)
sim_dir = os.path.join(input_folder, 'test_sim')
if not os.path.isdir(sim_dir):
os.makedirs(sim_dir)
simulator = SimulatorAdapterModel()
simulator.connectivity = connectivity_factory().gid
h5.store_view_model(simulator, sim_dir)
zip_filename = os.path.join(input_folder,
RequestFileKey.SIMULATION_FILE_NAME.value)
self.storage_interface.zip_folder(zip_filename, sim_dir)
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request')
fp = open(zip_filename, 'rb')
request_mock.files = {
RequestFileKey.SIMULATION_FILE_KEY.value:
FileStorage(fp, os.path.basename(zip_filename))
}
def launch_sim(self, user_id, project, algorithm, zip_folder_path,
simulator_file):
return Operation('', '', '', {})
# Mock simulation launch and current user
mocker.patch.object(SimulatorService, 'prepare_simulation_on_server',
launch_sim)
operation_gid, status = self.simulation_resource.post(
project_gid=self.test_project.gid)
fp.close()
assert type(operation_gid) is str
assert status == 201
def transactional_teardown_method(self):
self.storage_interface.remove_project_structure(self.test_project.name)
class TestProjectService(TransactionalTestCase):
"""
This class contains tests for the tvb.core.services.project_service module.
"""
def transactional_setup_method(self):
"""
Reset the database before each test.
"""
self.project_service = ProjectService()
self.storage_interface = StorageInterface()
self.test_user = TestFactory.create_user()
def transactional_teardown_method(self):
"""
Remove project folders.
"""
self.delete_project_folders()
def test_create_project_happy_flow(self):
user1 = TestFactory.create_user('test_user1')
user2 = TestFactory.create_user('test_user2')
initial_projects = dao.get_projects_for_user(self.test_user.id)
assert len(initial_projects) == 0, "Database reset probably failed!"
TestFactory.create_project(self.test_user, 'test_project', "description", users=[user1.id, user2.id])
resulting_projects = dao.get_projects_for_user(self.test_user.id)
assert len(resulting_projects) == 1, "Project with valid data not inserted!"
project = resulting_projects[0]
assert project.name == "test_project", "Invalid retrieved project name"
assert project.description == "description", "Description do no match"
users_for_project = dao.get_members_of_project(project.id)
for user in users_for_project:
assert user.id in [user1.id, user2.id, self.test_user.id], "Users not stored properly."
assert os.path.exists(os.path.join(TvbProfile.current.TVB_STORAGE,
StorageInterface.PROJECTS_FOLDER,
"test_project")), "Folder for project was not created"
def test_create_project_empty_name(self):
"""
Creating a project with an empty name.
"""
data = dict(name="", description="test_description", users=[])
initial_projects = dao.get_projects_for_user(self.test_user.id)
assert len(initial_projects) == 0, "Database reset probably failed!"
with pytest.raises(ProjectServiceException):
self.project_service.store_project(self.test_user, True, None, **data)
def test_edit_project_happy_flow(self):
"""
Standard flow for editing an existing project.
"""
selected_project = TestFactory.create_project(self.test_user, 'test_proj')
proj_root = self.storage_interface.get_project_folder(selected_project.name)
initial_projects = dao.get_projects_for_user(self.test_user.id)
assert len(initial_projects) == 1, "Database initialization probably failed!"
edited_data = dict(name="test_project", description="test_description", users=[])
edited_project = self.project_service.store_project(self.test_user, False, selected_project.id, **edited_data)
assert not os.path.exists(proj_root), "Previous folder not deleted"
proj_root = self.storage_interface.get_project_folder(edited_project.name)
assert os.path.exists(proj_root), "New folder not created!"
assert selected_project.name != edited_project.name, "Project was no changed!"
def test_edit_project_unexisting(self):
"""
Trying to edit an un-existing project.
"""
selected_project = TestFactory.create_project(self.test_user, 'test_proj')
self.storage_interface.get_project_folder(selected_project.name)
initial_projects = dao.get_projects_for_user(self.test_user.id)
assert len(initial_projects) == 1, "Database initialization probably failed!"
data = dict(name="test_project", description="test_description", users=[])
with pytest.raises(ProjectServiceException):
self.project_service.store_project(self.test_user, False, 99, **data)
def test_find_project_happy_flow(self):
"""
Standard flow for finding a project by it's id.
"""
initial_projects = dao.get_projects_for_user(self.test_user.id)
assert len(initial_projects) == 0, "Database reset probably failed!"
inserted_project = TestFactory.create_project(self.test_user, 'test_project')
assert self.project_service.find_project(inserted_project.id) is not None, "Project not found !"
dao_returned_project = dao.get_project_by_id(inserted_project.id)
service_returned_project = self.project_service.find_project(inserted_project.id)
assert dao_returned_project.id == service_returned_project.id, \
"Data returned from service is different from data returned by DAO."
assert dao_returned_project.name == service_returned_project.name, \
"Data returned from service is different than data returned by DAO."
assert dao_returned_project.description == service_returned_project.description, \
"Data returned from service is different from data returned by DAO."
assert dao_returned_project.members == service_returned_project.members, \
"Data returned from service is different from data returned by DAO."
def test_find_project_unexisting(self):
"""
Searching for an un-existing project.
"""
data = dict(name="test_project", description="test_description", users=[])
initial_projects = dao.get_projects_for_user(self.test_user.id)
assert len(initial_projects) == 0, "Database reset probably failed!"
project = self.project_service.store_project(self.test_user, True, None, **data)
# fetch a likely non-existing project. Previous project id plus a 'big' offset
with pytest.raises(ProjectServiceException):
self.project_service.find_project(project.id + 1033)
def test_retrieve_projects_for_user(self):
"""
Test for retrieving the projects for a given user. One page only.
"""
initial_projects = self.project_service.retrieve_projects_for_user(self.test_user.id)[0]
assert len(initial_projects) == 0, "Database was not reset properly!"
TestFactory.create_project(self.test_user, 'test_proj')
TestFactory.create_project(self.test_user, 'test_proj1')
TestFactory.create_project(self.test_user, 'test_proj2')
user1 = TestFactory.create_user('another_user')
TestFactory.create_project(user1, 'test_proj3')
projects = self.project_service.retrieve_projects_for_user(self.test_user.id)[0]
assert len(projects) == 3, "Projects not retrieved properly!"
for project in projects:
assert project.name != "test_project3", "This project should not have been retrieved"
def test_retrieve_1project_3usr(self):
"""
One user as admin, two users as members, getting projects for admin and for any of
the members should return one.
"""
member1 = TestFactory.create_user("member1")
member2 = TestFactory.create_user("member2")
TestFactory.create_project(self.test_user, 'Testproject', users=[member1.id, member2.id])
projects = self.project_service.retrieve_projects_for_user(self.test_user.id, 1)[0]
assert len(projects) == 1, "Projects not retrieved properly!"
projects = self.project_service.retrieve_projects_for_user(member1.id, 1)[0]
assert len(projects) == 1, "Projects not retrieved properly!"
projects = self.project_service.retrieve_projects_for_user(member2.id, 1)[0]
assert len(projects) == 1, "Projects not retrieved properly!"
def test_retrieve_3projects_3usr(self):
"""
Three users, 3 projects. Structure of db:
proj1: {admin: user1, members: [user2, user3]}
proj2: {admin: user2, members: [user1]}
proj3: {admin: user3, members: [user1, user2]}
Check valid project returns for all the users.
"""
member1 = TestFactory.create_user("member1")
member2 = TestFactory.create_user("member2")
member3 = TestFactory.create_user("member3")
TestFactory.create_project(member1, 'TestProject1', users=[member2.id, member3.id])
TestFactory.create_project(member2, 'TestProject2', users=[member1.id])
TestFactory.create_project(member3, 'TestProject3', users=[member1.id, member2.id])
projects = self.project_service.retrieve_projects_for_user(member1.id, 1)[0]
assert len(projects) == 3, "Projects not retrieved properly!"
projects = self.project_service.retrieve_projects_for_user(member2.id, 1)[0]
assert len(projects) == 3, "Projects not retrieved properly!"
projects = self.project_service.retrieve_projects_for_user(member3.id, 1)[0]
assert len(projects) == 2, "Projects not retrieved properly!"
def test_retrieve_projects_random(self):
"""
Generate a large number of users/projects, and validate the results.
"""
ExtremeTestFactory.generate_users(NR_USERS, MAX_PROJ_PER_USER)
for i in range(NR_USERS):
current_user = dao.get_user_by_name("gen" + str(i))
expected_projects = ExtremeTestFactory.VALIDATION_DICT[current_user.id]
if expected_projects % PROJECTS_PAGE_SIZE == 0:
expected_pages = expected_projects / PROJECTS_PAGE_SIZE
exp_proj_per_page = PROJECTS_PAGE_SIZE
else:
expected_pages = expected_projects // PROJECTS_PAGE_SIZE + 1
exp_proj_per_page = expected_projects % PROJECTS_PAGE_SIZE
if expected_projects == 0:
expected_pages = 0
exp_proj_per_page = 0
projects, pages = self.project_service.retrieve_projects_for_user(current_user.id, expected_pages)
assert len(projects) == exp_proj_per_page, "Projects not retrieved properly! Expected:" + \
str(exp_proj_per_page) + "but got:" + str(len(projects))
assert pages == expected_pages, "Pages not retrieved properly!"
for folder in os.listdir(TvbProfile.current.TVB_STORAGE):
full_path = os.path.join(TvbProfile.current.TVB_STORAGE, folder)
if folder.startswith('Generated'):
self.storage_interface.remove_folder(full_path)
def test_retrieve_projects_page2(self):
"""
Test for retrieving the second page projects for a given user.
"""
for i in range(PROJECTS_PAGE_SIZE + 3):
TestFactory.create_project(self.test_user, 'test_proj' + str(i))
projects, pages = self.project_service.retrieve_projects_for_user(self.test_user.id, 2)
assert len(projects) == (PROJECTS_PAGE_SIZE + 3) % PROJECTS_PAGE_SIZE, "Pagination inproper."
assert pages == 2, 'Wrong number of pages retrieved.'
def test_retrieve_projects_and_del(self):
"""
Test for retrieving the second page projects for a given user.
"""
created_projects = []
for i in range(PROJECTS_PAGE_SIZE + 1):
created_projects.append(TestFactory.create_project(self.test_user, 'test_proj' + str(i)))
projects, pages = self.project_service.retrieve_projects_for_user(self.test_user.id, 2)
assert len(projects) == (PROJECTS_PAGE_SIZE + 1) % PROJECTS_PAGE_SIZE, "Pagination improper."
assert pages == (PROJECTS_PAGE_SIZE + 1) // PROJECTS_PAGE_SIZE + 1, 'Wrong number of pages'
self.project_service.remove_project(created_projects[1].id)
projects, pages = self.project_service.retrieve_projects_for_user(self.test_user.id, 2)
assert len(projects) == 0, "Pagination improper."
assert pages == 1, 'Wrong number of pages retrieved.'
projects, pages = self.project_service.retrieve_projects_for_user(self.test_user.id, 1)
assert len(projects) == PROJECTS_PAGE_SIZE, "Pagination improper."
assert pages == 1, 'Wrong number of pages retrieved.'
def test_empty_project_has_zero_disk_size(self):
TestFactory.create_project(self.test_user, 'test_proj')
projects, pages = self.project_service.retrieve_projects_for_user(self.test_user.id)
assert 0 == projects[0].disk_size
assert '0.0 KiB' == projects[0].disk_size_human
def test_project_disk_size(self):
project1 = TestFactory.create_project(self.test_user, 'test_proj1')
zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_66.zip')
TestFactory.import_zip_connectivity(self.test_user, project1, zip_path, 'testSubject')
project2 = TestFactory.create_project(self.test_user, 'test_proj2')
zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_76.zip')
TestFactory.import_zip_connectivity(self.test_user, project2, zip_path, 'testSubject')
projects = self.project_service.retrieve_projects_for_user(self.test_user.id)[0]
assert projects[0].disk_size != projects[1].disk_size, "projects should have different size"
for project in projects:
assert 0 != project.disk_size
assert '0.0 KiB' != project.disk_size_human
prj_folder = self.storage_interface.get_project_folder(project.name)
actual_disk_size = self.storage_interface.compute_recursive_h5_disk_usage(prj_folder)
ratio = float(actual_disk_size) / project.disk_size
msg = "Real disk usage: %s The one recorded in the db : %s" % (actual_disk_size, project.disk_size)
assert ratio < 1.1, msg
def test_get_linkable_projects(self):
"""
Test for retrieving the projects for a given user.
"""
initial_projects = self.project_service.retrieve_projects_for_user(self.test_user.id)[0]
assert len(initial_projects) == 0, "Database was not reset!"
test_proj = []
user1 = TestFactory.create_user("another_user")
for i in range(4):
test_proj.append(TestFactory.create_project(self.test_user if i < 3 else user1, 'test_proj' + str(i)))
operation = TestFactory.create_operation(test_user=self.test_user, test_project=test_proj[0])
datatype = dao.store_entity(model_datatype.DataType(module="test_data", subject="subj1",
state="test_state", operation_id=operation.id))
linkable = self.project_service.get_linkable_projects_for_user(self.test_user.id, str(datatype.id))[0]
assert len(linkable) == 2, "Wrong count of link-able projects!"
proj_names = [project.name for project in linkable]
assert test_proj[1].name in proj_names
assert test_proj[2].name in proj_names
assert not test_proj[3].name in proj_names
def test_remove_project_happy_flow(self):
"""
Standard flow for deleting a project.
"""
inserted_project = TestFactory.create_project(self.test_user, 'test_proj')
project_root = self.storage_interface.get_project_folder(inserted_project.name)
projects = dao.get_projects_for_user(self.test_user.id)
assert len(projects) == 1, "Initializations failed!"
assert os.path.exists(project_root), "Something failed at insert time!"
self.project_service.remove_project(inserted_project.id)
projects = dao.get_projects_for_user(self.test_user.id)
assert len(projects) == 0, "Project was not deleted!"
assert not os.path.exists(project_root), "Root folder not deleted!"
def test_remove_project_wrong_id(self):
"""
Flow for deleting a project giving an un-existing id.
"""
TestFactory.create_project(self.test_user, 'test_proj')
projects = dao.get_projects_for_user(self.test_user.id)
assert len(projects) == 1, "Initializations failed!"
with pytest.raises(ProjectServiceException):
self.project_service.remove_project(99)
def __check_meta_data(self, expected_meta_data, new_datatype):
"""Validate Meta-Data"""
mapp_keys = {DataTypeOverlayDetails.DATA_SUBJECT: "subject", DataTypeOverlayDetails.DATA_STATE: "state"}
for key, value in expected_meta_data.items():
if key in mapp_keys:
assert value == getattr(new_datatype, mapp_keys[key])
elif key == DataTypeMetaData.KEY_OPERATION_TAG:
if DataTypeMetaData.KEY_OP_GROUP_ID in expected_meta_data:
# We have a Group to check
op_group = new_datatype.parent_operation.fk_operation_group
op_group = dao.get_generic_entity(model_operation.OperationGroup, op_group)[0]
assert value == op_group.name
else:
assert value == new_datatype.parent_operation.user_group
def test_remove_project_node(self):
"""
Test removing of a node from a project.
"""
inserted_project, gid, op = TestFactory.create_value_wrapper(self.test_user)
project_to_link = model_project.Project("Link", self.test_user.id, "descript")
project_to_link = dao.store_entity(project_to_link)
exact_data = dao.get_datatype_by_gid(gid)
assert exact_data is not None, "Initialization problem!"
link = dao.store_entity(model_datatype.Links(exact_data.id, project_to_link.id))
vw_h5_path = h5.path_for_stored_index(exact_data)
assert os.path.exists(vw_h5_path)
if dao.get_system_user() is None:
dao.store_entity(model_operation.User(TvbProfile.current.web.admin.SYSTEM_USER_NAME,
TvbProfile.current.web.admin.SYSTEM_USER_NAME, None, None, True,
None))
self.project_service._remove_project_node_files(inserted_project.id, gid, [link])
assert not os.path.exists(vw_h5_path)
exact_data = dao.get_datatype_by_gid(gid)
assert exact_data is not None, "Data should still be in DB, because of links"
vw_h5_path_new = h5.path_for_stored_index(exact_data)
assert os.path.exists(vw_h5_path_new)
assert vw_h5_path_new != vw_h5_path
self.project_service._remove_project_node_files(project_to_link.id, gid, [])
assert dao.get_datatype_by_gid(gid) is None
def test_update_meta_data_simple(self):
"""
Test the new update metaData for a simple data that is not part of a group.
"""
inserted_project, gid, _ = TestFactory.create_value_wrapper(self.test_user)
new_meta_data = {DataTypeOverlayDetails.DATA_SUBJECT: "new subject",
DataTypeOverlayDetails.DATA_STATE: "second_state",
DataTypeOverlayDetails.CODE_GID: gid,
DataTypeOverlayDetails.CODE_OPERATION_TAG: 'new user group'}
self.project_service.update_metadata(new_meta_data)
new_datatype = dao.get_datatype_by_gid(gid)
self.__check_meta_data(new_meta_data, new_datatype)
new_datatype_h5 = h5.h5_file_for_index(new_datatype)
assert new_datatype_h5.subject.load() == 'new subject', 'UserGroup not updated!'
def test_update_meta_data_group(self, test_adapter_factory, datatype_group_factory):
"""
Test the new update metaData for a group of dataTypes.
"""
test_adapter_factory(adapter_class=DummyAdapter3)
group, _ = datatype_group_factory()
op_group_id = group.fk_operation_group
new_meta_data = {DataTypeOverlayDetails.DATA_SUBJECT: "new subject",
DataTypeOverlayDetails.DATA_STATE: "updated_state",
DataTypeOverlayDetails.CODE_OPERATION_GROUP_ID: op_group_id,
DataTypeOverlayDetails.CODE_OPERATION_TAG: 'newGroupName'}
self.project_service.update_metadata(new_meta_data)
datatypes = dao.get_datatype_in_group(op_group_id)
for datatype in datatypes:
new_datatype = dao.get_datatype_by_id(datatype.id)
assert op_group_id == new_datatype.parent_operation.fk_operation_group
new_group = dao.get_generic_entity(model_operation.OperationGroup, op_group_id)[0]
assert new_group.name == "newGroupName"
self.__check_meta_data(new_meta_data, new_datatype)
def test_retrieve_project_full(self, dummy_datatype_index_factory):
"""
Tests full project information is retrieved by method `ProjectService.retrieve_project_full(...)`
"""
project = TestFactory.create_project(self.test_user)
operation = TestFactory.create_operation(test_user=self.test_user, test_project=project)
dummy_datatype_index_factory(project=project, operation=operation)
dummy_datatype_index_factory(project=project, operation=operation)
dummy_datatype_index_factory(project=project, operation=operation)
_, ops_nr, operations, pages_no = self.project_service.retrieve_project_full(project.id)
assert ops_nr == 1, "DataType Factory should only use one operation to store all it's datatypes."
assert pages_no == 1, "DataType Factory should only use one operation to store all it's datatypes."
resulted_dts = operations[0]['results']
assert len(resulted_dts) == 3, "3 datatypes should be created."
def test_get_project_structure(self, datatype_group_factory, dummy_datatype_index_factory,
project_factory, user_factory):
"""
Tests project structure is as expected and contains all datatypes and created links
"""
user = user_factory()
project1 = project_factory(user, name="TestPS1")
project2 = project_factory(user, name="TestPS2")
dt_group, _ = datatype_group_factory(project=project1)
dt_simple = dummy_datatype_index_factory(state="RAW_DATA", project=project1)
# Create 3 DTs directly in Project 2
dummy_datatype_index_factory(state="RAW_DATA", project=project2)
dummy_datatype_index_factory(state="RAW_DATA", project=project2)
dummy_datatype_index_factory(state="RAW_DATA", project=project2)
# Create Links from Project 1 into Project 2
link_ids, expected_links = [], []
link_ids.append(dt_simple.id)
expected_links.append(dt_simple.gid)
# Prepare links towards a full DT Group, but expecting only the DT_Group in the final tree
dts = dao.get_datatype_in_group(datatype_group_id=dt_group.id)
link_ids.extend([dt_to_link.id for dt_to_link in dts])
link_ids.append(dt_group.id)
expected_links.append(dt_group.gid)
# Actually create the links from Prj1 into Prj2
for link_id in link_ids:
AlgorithmService().create_link(link_id, project2.id)
# Retrieve the raw data used to compose the tree (for easy parsing)
dts_in_tree = dao.get_data_in_project(project2.id)
dts_in_tree = [dt.gid for dt in dts_in_tree]
# Retrieve the tree json (for trivial validations only, as we can not decode)
node_json = self.project_service.get_project_structure(project2, None, DataTypeMetaData.KEY_STATE,
DataTypeMetaData.KEY_SUBJECT, None)
assert len(expected_links) + 3 == len(dts_in_tree), "invalid number of nodes in tree"
assert dt_group.gid in dts_in_tree, "DT_Group should be in the Project Tree!"
assert dt_group.gid in node_json, "DT_Group should be in the Project Tree JSON!"
project_dts = dao.get_datatypes_in_project(project2.id)
for dt in project_dts:
if dt.fk_datatype_group is not None:
assert not dt.gid in node_json, "DTs part of a group should not be"
assert not dt.gid in dts_in_tree, "DTs part of a group should not be"
else:
assert dt.gid in node_json, "Simple DTs and DT_Groups should be"
assert dt.gid in dts_in_tree, "Simple DTs and DT_Groups should be"
for link_gid in expected_links:
assert link_gid in node_json, "Expected Link not present"
assert link_gid in dts_in_tree, "Expected Link not present"
class TestOperationResource(RestResourceTest):
def transactional_setup_method(self):
self.test_user = TestFactory.create_user('Rest_User')
self.test_project = TestFactory.create_project(self.test_user, 'Rest_Project', users=[self.test_user.id])
self.operations_resource = GetOperationsInProjectResource()
self.status_resource = GetOperationStatusResource()
self.results_resource = GetOperationResultsResource()
self.launch_resource = LaunchOperationResource()
self.storage_interface = StorageInterface()
def test_server_get_operation_status_inexistent_gid(self, mocker):
self._mock_user(mocker)
operation_gid = "inexistent-gid"
with pytest.raises(InvalidIdentifierException): self.status_resource.get(operation_gid=operation_gid)
def test_server_get_operation_status(self, mocker):
self._mock_user(mocker)
zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_96.zip')
TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path)
request_mock = mocker.patch.object(flask, 'request', spec={})
request_mock.args = {Strings.PAGE_NUMBER: '1'}
operations_and_pages = self.operations_resource.get(project_gid=self.test_project.gid)
result = self.status_resource.get(operation_gid=operations_and_pages['operations'][0].gid)
assert type(result) is str
assert result in OperationPossibleStatus
def test_server_get_operation_results_inexistent_gid(self, mocker):
self._mock_user(mocker)
operation_gid = "inexistent-gid"
with pytest.raises(InvalidIdentifierException): self.results_resource.get(operation_gid=operation_gid)
def test_server_get_operation_results(self, mocker):
self._mock_user(mocker)
zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_96.zip')
TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path)
request_mock = mocker.patch.object(flask, 'request', spec={})
request_mock.args = {Strings.PAGE_NUMBER: '1'}
operations_and_pages = self.operations_resource.get(project_gid=self.test_project.gid)
result = self.results_resource.get(operation_gid=operations_and_pages['operations'][0].gid)
assert type(result) is list
assert len(result) == 1
def test_server_get_operation_results_failed_operation(self, mocker):
self._mock_user(mocker)
zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_90.zip')
with pytest.raises(TVBException):
TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path)
request_mock = mocker.patch.object(flask, 'request', spec={})
request_mock.args = {Strings.PAGE_NUMBER: '1'}
operations_and_pages = self.operations_resource.get(project_gid=self.test_project.gid)
result = self.results_resource.get(operation_gid=operations_and_pages['operations'][0].gid)
assert type(result) is list
assert len(result) == 0
def test_server_launch_operation_no_file(self, mocker):
self._mock_user(mocker)
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request', spec={})
request_mock.files = {}
with pytest.raises(InvalidIdentifierException): self.launch_resource.post(project_gid='', algorithm_module='',
algorithm_classname='')
def test_server_launch_operation_wrong_file_extension(self, mocker):
self._mock_user(mocker)
dummy_file = FileStorage(BytesIO(b"test"), 'test.txt')
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request', spec={})
request_mock.files = {RequestFileKey.LAUNCH_ANALYZERS_MODEL_FILE.value: dummy_file}
with pytest.raises(InvalidIdentifierException): self.launch_resource.post(project_gid='', algorithm_module='',
algorithm_classname='')
def test_server_launch_operation_inexistent_gid(self, mocker):
self._mock_user(mocker)
project_gid = "inexistent-gid"
dummy_file = FileStorage(BytesIO(b"test"), 'test.h5')
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request', spec={})
request_mock.files = {RequestFileKey.LAUNCH_ANALYZERS_MODEL_FILE.value: dummy_file}
with pytest.raises(InvalidIdentifierException): self.launch_resource.post(project_gid=project_gid,
algorithm_module='',
algorithm_classname='')
def test_server_launch_operation_inexistent_algorithm(self, mocker):
self._mock_user(mocker)
inexistent_algorithm = "inexistent-algorithm"
dummy_file = FileStorage(BytesIO(b"test"), 'test.h5')
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request', spec={})
request_mock.files = {RequestFileKey.LAUNCH_ANALYZERS_MODEL_FILE.value: dummy_file}
with pytest.raises(ServiceException): self.launch_resource.post(project_gid=self.test_project.gid,
algorithm_module=inexistent_algorithm,
algorithm_classname='')
def test_server_launch_operation(self, mocker, time_series_index_factory):
self._mock_user(mocker)
algorithm_module = "tvb.adapters.analyzers.fourier_adapter"
algorithm_class = "FourierAdapter"
input_ts_index = time_series_index_factory()
fft_model = FFTAdapterModel()
fft_model.time_series = UUID(input_ts_index.gid)
fft_model.window_function = list(SUPPORTED_WINDOWING_FUNCTIONS)[0]
input_folder = self.storage_interface.get_project_folder(self.test_project.name)
view_model_h5_path = h5.store_view_model(fft_model, input_folder)
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request', spec={})
fp = open(view_model_h5_path, 'rb')
request_mock.files = {
RequestFileKey.LAUNCH_ANALYZERS_MODEL_FILE.value: FileStorage(fp, os.path.basename(view_model_h5_path))}
# Mock launch_operation() call and current_user
mocker.patch.object(OperationService, 'launch_operation')
operation_gid, status = self.launch_resource.post(project_gid=self.test_project.gid,
algorithm_module=algorithm_module,
algorithm_classname=algorithm_class)
fp.close()
assert type(operation_gid) is str
assert len(operation_gid) > 0
class ABCAdapter(object):
"""
Root Abstract class for all TVB Adapters.
"""
# model.Algorithm instance that will be set for each adapter class created by in build_adapter method
stored_adapter = None
launch_mode = AdapterLaunchModeEnum.ASYNC_DIFF_MEM
def __init__(self):
self.generic_attributes = GenericAttributes()
self.generic_attributes.subject = DataTypeMetaData.DEFAULT_SUBJECT
self.storage_interface = StorageInterface()
# Will be populate with current running operation's identifier
self.operation_id = None
self.user_id = None
self.submitted_form = None
self.log = get_logger(self.__class__.__module__)
@classmethod
def get_group_name(cls):
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "name"):
return cls._ui_group.name
return None
@classmethod
def get_group_description(cls):
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "description"):
return cls._ui_group.description
return None
@classmethod
def get_ui_name(cls):
if hasattr(cls, "_ui_name"):
return cls._ui_name
else:
return cls.__name__
@classmethod
def get_ui_description(cls):
if hasattr(cls, "_ui_description"):
return cls._ui_description
@classmethod
def get_ui_subsection(cls):
if hasattr(cls, "_ui_subsection"):
return cls._ui_subsection
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "subsection"):
return cls._ui_group.subsection
@staticmethod
def can_be_active():
"""
To be overridden where needed (e.g. Matlab dependent adapters).
:return: By default True, and False when the current Adapter can not be executed in the current env
for various reasons (e.g. no Matlab or Octave installed)
"""
return True
def submit_form(self, form):
self.submitted_form = form
# TODO separate usage of get_form_class (returning a class) and return of a submitted instance
def get_form(self):
if self.submitted_form is not None:
return self.submitted_form
return self.get_form_class()
@abstractmethod
def get_form_class(self):
return None
def get_adapter_fragments(self, view_model):
"""
The result will be used for introspecting and checking operation changed input
params from the defaults, to show in web gui.
:return: a list of ABCAdapterForm classes, in case the current Adapter GUI
will be composed of multiple sub-forms.
"""
return {}
def get_view_model_class(self):
return self.get_form_class().get_view_model()
@abstractmethod
def get_output(self):
"""
Describes inputs and outputs of the launch method.
"""
def configure(self, view_model):
"""
To be implemented in each Adapter that requires any specific configurations
before the actual launch.
"""
@abstractmethod
def get_required_memory_size(self, view_model):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter.
"""
@abstractmethod
def get_required_disk_size(self, view_model):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter in kilo-Bytes.
"""
def get_execution_time_approximation(self, view_model):
"""
Method should approximate based on input arguments, the time it will take for the operation
to finish (in seconds).
"""
return -1
@abstractmethod
def launch(self, view_model):
"""
To be implemented in each Adapter.
Will contain the logic of the Adapter.
Takes a ViewModel with data, dependency direction is: Adapter -> Form -> ViewModel
Any returned DataType will be stored in DB, by the Framework.
:param view_model: the data model corresponding to the current adapter
"""
def add_operation_additional_info(self, message):
"""
Adds additional info on the operation to be displayed in the UI. Usually a warning message.
"""
current_op = dao.get_operation_by_id(self.operation_id)
current_op.additional_info = message
dao.store_entity(current_op)
def extract_operation_data(self, operation):
operation = dao.get_operation_by_id(operation.id)
self.operation_id = operation.id
self.current_project_id = operation.project.id
self.user_id = operation.fk_launched_by
def _ensure_enough_resources(self, available_disk_space, view_model):
# Compare the amount of memory the current algorithms states it needs,
# with the average between the RAM available on the OS and the free memory at the current moment.
# We do not consider only the free memory, because some OSs are freeing late and on-demand only.
total_free_memory = psutil.virtual_memory().free + psutil.swap_memory().free
total_existent_memory = psutil.virtual_memory().total + psutil.swap_memory().total
memory_reference = (total_free_memory + total_existent_memory) / 2
adapter_required_memory = self.get_required_memory_size(view_model)
if adapter_required_memory > memory_reference:
msg = "Machine does not have enough RAM memory for the operation (expected %.2g GB, but found %.2g GB)."
raise NoMemoryAvailableException(msg % (adapter_required_memory / 2 ** 30, memory_reference / 2 ** 30))
# Compare the expected size of the operation results with the HDD space currently available for the user
# TVB defines a quota per user.
required_disk_space = self.get_required_disk_size(view_model)
if available_disk_space < 0:
msg = "You have exceeded you HDD space quota by %.2f MB Stopping execution."
raise NoMemoryAvailableException(msg % (- available_disk_space / 2 ** 10))
if available_disk_space < required_disk_space:
msg = ("You only have %.2f GB of disk space available but the operation you "
"launched might require %.2f Stopping execution...")
raise NoMemoryAvailableException(msg % (available_disk_space / 2 ** 20, required_disk_space / 2 ** 20))
return required_disk_space
def _update_operation_entity(self, operation, required_disk_space):
operation.start_now()
operation.estimated_disk_size = required_disk_space
dao.store_entity(operation)
@nan_not_allowed()
def _prelaunch(self, operation, view_model, available_disk_space=0):
"""
Method to wrap LAUNCH.
Will prepare data, and store results on return.
"""
self.extract_operation_data(operation)
self.generic_attributes.fill_from(view_model.generic_attributes)
self.configure(view_model)
required_disk_size = self._ensure_enough_resources(available_disk_space, view_model)
self._update_operation_entity(operation, required_disk_size)
result = self.launch(view_model)
if not isinstance(result, (list, tuple)):
result = [result, ]
self.__check_integrity(result)
return self._capture_operation_results(result)
def _capture_operation_results(self, result):
"""
After an operation was finished, make sure the results are stored
in DB storage and the correct meta-data,IDs are set.
"""
data_type_group_id = None
operation = dao.get_operation_by_id(self.operation_id)
if operation.user_group is None or len(operation.user_group) == 0:
operation.user_group = date2string(datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT)
operation = dao.store_entity(operation)
if self._is_group_launch():
data_type_group_id = dao.get_datatypegroup_by_op_group_id(operation.fk_operation_group).id
count_stored = 0
if result is None:
return "", count_stored
group_type = None # In case of a group, the first not-none type is sufficient to memorize here
for res in result:
if res is None:
continue
if not res.fixed_generic_attributes:
res.fill_from_generic_attributes(self.generic_attributes)
res.fk_from_operation = self.operation_id
res.fk_datatype_group = data_type_group_id
associated_file = h5.path_for_stored_index(res)
if os.path.exists(associated_file):
if not res.fixed_generic_attributes:
with H5File.from_file(associated_file) as f:
f.store_generic_attributes(self.generic_attributes)
# Compute size-on disk, in case file-storage is used
res.disk_size = self.storage_interface.compute_size_on_disk(associated_file)
dao.store_entity(res)
res.after_store()
group_type = res.type
count_stored += 1
if count_stored > 0 and self._is_group_launch():
# Update the operation group name
operation_group = dao.get_operationgroup_by_id(operation.fk_operation_group)
operation_group.fill_operationgroup_name(group_type)
dao.store_entity(operation_group)
return 'Operation ' + str(self.operation_id) + ' has finished.', count_stored
def __check_integrity(self, result):
"""
Check that the returned parameters for LAUNCH operation
are of the type specified in the adapter's interface.
"""
for result_entity in result:
if result_entity is None:
continue
if not self.__is_data_in_supported_types(result_entity):
msg = "Unexpected output DataType %s"
raise InvalidParameterException(msg % type(result_entity))
def __is_data_in_supported_types(self, data):
if data is None:
return True
for supported_type in self.get_output():
if isinstance(data, supported_type):
return True
# Data can't be mapped on any supported type !!
return False
def _is_group_launch(self):
"""
Return true if this adapter is launched from a group of operations
"""
operation = dao.get_operation_by_id(self.operation_id)
return operation.fk_operation_group is not None
def load_entity_by_gid(self, data_gid):
# type: (typing.Union[uuid.UUID, str]) -> DataType
"""
Load a generic DataType, specified by GID.
"""
idx = load_entity_by_gid(data_gid)
if idx and self.generic_attributes.parent_burst is None:
# Only in case the BurstConfiguration references hasn't been set already, take it from the current DT
self.generic_attributes.parent_burst = idx.fk_parent_burst
return idx
def load_traited_by_gid(self, data_gid):
# type: (typing.Union[uuid.UUID, str]) -> HasTraits
"""
Load a generic HasTraits instance, specified by GID.
"""
index = self.load_entity_by_gid(data_gid)
return h5.load_from_index(index)
def load_with_references(self, dt_gid):
# type: (typing.Union[uuid.UUID, str]) -> HasTraits
dt_index = self.load_entity_by_gid(dt_gid)
h5_path = h5.path_for_stored_index(dt_index)
dt, _ = h5.load_with_references(h5_path)
return dt
def view_model_to_has_traits(self, view_model):
# type: (ViewModel) -> HasTraits
has_traits_class = view_model.linked_has_traits
has_traits = has_traits_class()
view_model_class = type(view_model)
if not has_traits_class:
raise Exception("There is no linked HasTraits for this ViewModel {}".format(type(view_model)))
for attr_name in has_traits_class.declarative_attrs:
view_model_class_attr = getattr(view_model_class, attr_name)
view_model_attr = getattr(view_model, attr_name)
if isinstance(view_model_class_attr, DataTypeGidAttr) and view_model_attr:
attr_value = self.load_with_references(view_model_attr)
elif isinstance(view_model_class_attr, Attr) and isinstance(view_model_attr, ViewModel):
attr_value = self.view_model_to_has_traits(view_model_attr)
elif isinstance(view_model_class_attr, List) and len(view_model_attr) > 0 and isinstance(view_model_attr[0],
ViewModel):
attr_value = list()
for view_model_elem in view_model_attr:
elem = self.view_model_to_has_traits(view_model_elem)
attr_value.append(elem)
else:
attr_value = view_model_attr
setattr(has_traits, attr_name, attr_value)
return has_traits
@staticmethod
def build_adapter_from_class(adapter_class):
"""
Having a subclass of ABCAdapter, prepare an instance for launching an operation with it.
"""
if not issubclass(adapter_class, ABCAdapter):
raise IntrospectionException("Invalid data type: It should extend adapters.ABCAdapter!")
try:
stored_adapter = dao.get_algorithm_by_module(adapter_class.__module__, adapter_class.__name__)
adapter_instance = adapter_class()
adapter_instance.stored_adapter = stored_adapter
return adapter_instance
except Exception as excep:
LOGGER.exception(excep)
raise IntrospectionException(str(excep))
@staticmethod
def determine_adapter_class(stored_adapter):
# type: (Algorithm) -> ABCAdapter
"""
Determine the class of an adapter based on module and classname strings from stored_adapter
:param stored_adapter: Algorithm or AlgorithmDTO type
:return: a subclass of ABCAdapter
"""
ad_module = importlib.import_module(stored_adapter.module)
adapter_class = getattr(ad_module, stored_adapter.classname)
return adapter_class
@staticmethod
def build_adapter(stored_adapter):
# type: (Algorithm) -> ABCAdapter
"""
Having a module and a class name, create an instance of ABCAdapter.
"""
try:
adapter_class = ABCAdapter.determine_adapter_class(stored_adapter)
adapter_instance = adapter_class()
adapter_instance.stored_adapter = stored_adapter
return adapter_instance
except Exception:
msg = "Could not load Adapter Instance for Stored row %s" % stored_adapter
LOGGER.exception(msg)
raise IntrospectionException(msg)
def load_view_model(self, operation):
storage_path = self.storage_interface.get_project_folder(operation.project.name, str(operation.id))
input_gid = operation.view_model_gid
return h5.load_view_model(input_gid, storage_path)
@staticmethod
def array_size2kb(size):
"""
:param size: size in bytes
:return: size in kB
"""
return size * TvbProfile.current.MAGIC_NUMBER / 8 / 2 ** 10
@staticmethod
def fill_index_from_h5(analyzer_index, analyzer_h5):
"""
Method used only by analyzers that write slices of data.
As they never have the whole array_data in memory, the metadata related to array_data (min, max, etc.) they
store on the index is not correct, so we need to update them.
"""
metadata = analyzer_h5.array_data.get_cached_metadata()
if not metadata.has_complex:
analyzer_index.array_data_max = float(metadata.max)
analyzer_index.array_data_min = float(metadata.min)
analyzer_index.array_data_mean = float(metadata.mean)
analyzer_index.aray_has_complex = metadata.has_complex
analyzer_index.array_is_finite = metadata.is_finite
analyzer_index.shape = json.dumps(analyzer_h5.array_data.shape)
analyzer_index.ndim = len(analyzer_h5.array_data.shape)
def path_for(self, h5_file_class, gid, dt_class=None):
project = dao.get_project_by_id(self.current_project_id)
return h5.path_for(self.operation_id, h5_file_class, gid, project.name, dt_class)
def store_complete(self, datatype, generic_attributes=GenericAttributes()):
project = dao.get_project_by_id(self.current_project_id)
return h5.store_complete(datatype, self.operation_id, project.name, generic_attributes)
def get_storage_path(self):
project = dao.get_project_by_id(self.current_project_id)
return self.storage_interface.get_project_folder(project.name, str(self.operation_id))
class ProjectService:
"""
Services layer for Project entities.
"""
def __init__(self):
self.logger = get_logger(__name__)
self.storage_interface = StorageInterface()
def store_project(self, current_user, is_create, selected_id, **data):
"""
We want to create/update a project entity.
"""
# Validate Unique Name
new_name = data["name"]
if len(new_name) < 1:
raise ProjectServiceException("Invalid project name!")
projects_no = dao.count_projects_for_name(new_name, selected_id)
if projects_no > 0:
err = {'name': 'Please choose another name, this one is used!'}
raise formencode.Invalid("Duplicate Name Error", {},
None,
error_dict=err)
started_operations = dao.get_operation_numbers(selected_id)[1]
if started_operations > 0:
raise ProjectServiceException(
"A project can not be renamed while operations are still running!"
)
if is_create:
current_proj = Project(new_name, current_user.id,
data["description"])
self.storage_interface.get_project_folder(current_proj.name)
else:
try:
current_proj = dao.get_project_by_id(selected_id)
except Exception as excep:
self.logger.exception("An error has occurred!")
raise ProjectServiceException(str(excep))
if current_proj.name != new_name:
self.storage_interface.rename_project(current_proj.name,
new_name)
current_proj.name = new_name
current_proj.description = data["description"]
# Commit to make sure we have a valid ID
current_proj.refresh_update_date()
_, metadata_proj = current_proj.to_dict()
self.storage_interface.write_project_metadata(metadata_proj)
current_proj = dao.store_entity(current_proj)
# Retrieve, to initialize lazy attributes
current_proj = dao.get_project_by_id(current_proj.id)
# Update share settings on current Project entity
visited_pages = []
prj_admin = current_proj.administrator.username
if 'visited_pages' in data and data['visited_pages']:
visited_pages = data['visited_pages'].split(',')
for page in visited_pages:
members = UserService.retrieve_users_except([prj_admin], int(page),
MEMBERS_PAGE_SIZE)[0]
members = [m.id for m in members]
dao.delete_members_for_project(current_proj.id, members)
selected_user_ids = data["users"]
if is_create and current_user.id not in selected_user_ids:
# Make the project admin also member of the current project
selected_user_ids.append(current_user.id)
dao.add_members_to_project(current_proj.id, selected_user_ids)
# Finish operation
self.logger.debug("Edit/Save OK for project:" + str(current_proj.id) +
' by user:'******'-'
result["count"] = one_op[2]
result["gid"] = one_op[13]
operation_group_id = one_op[3]
if operation_group_id is not None and operation_group_id:
try:
operation_group = dao.get_generic_entity(
OperationGroup, operation_group_id)[0]
result["group"] = operation_group.name
result["group"] = result["group"].replace("_", " ")
result["operation_group_id"] = operation_group.id
datatype_group = dao.get_datatypegroup_by_op_group_id(
operation_group_id)
result[
"datatype_group_gid"] = datatype_group.gid if datatype_group is not None else None
result["gid"] = operation_group.gid
# Filter only viewers for current DataTypeGroup entity:
if datatype_group is None:
view_groups = None
else:
view_groups = AlgorithmService(
).get_visualizers_for_group(datatype_group.gid)
result["view_groups"] = view_groups
except Exception:
self.logger.exception(
"We will ignore group on entity:" + str(one_op))
result["datatype_group_gid"] = None
else:
result['group'] = None
result['datatype_group_gid'] = None
result["algorithm"] = dao.get_algorithm_by_id(one_op[4])
result["user"] = dao.get_user_by_id(one_op[5])
if type(one_op[6]) is str:
result["create"] = string2date(str(one_op[6]))
else:
result["create"] = one_op[6]
if type(one_op[7]) is str:
result["start"] = string2date(str(one_op[7]))
else:
result["start"] = one_op[7]
if type(one_op[8]) is str:
result["complete"] = string2date(str(one_op[8]))
else:
result["complete"] = one_op[8]
if result["complete"] is not None and result[
"start"] is not None:
result["duration"] = format_timedelta(result["complete"] -
result["start"])
result["status"] = one_op[9]
result["additional"] = one_op[10]
result["visible"] = True if one_op[11] > 0 else False
result['operation_tag'] = one_op[12]
if not result['group']:
result['results'] = dao.get_results_for_operation(
result['id'])
else:
result['results'] = None
operations.append(result)
except Exception:
# We got an exception when processing one Operation Row. We will continue with the rest of the rows.
self.logger.exception(
"Could not prepare operation for display:" + str(one_op))
return selected_project, total_ops_nr, operations, pages_no
def retrieve_projects_for_user(self, user_id, current_page=1):
"""
Return a list with all Projects visible for current user.
"""
start_idx = PROJECTS_PAGE_SIZE * (current_page - 1)
total = dao.get_projects_for_user(user_id, is_count=True)
available_projects = dao.get_projects_for_user(user_id, start_idx,
PROJECTS_PAGE_SIZE)
pages_no = total // PROJECTS_PAGE_SIZE + (1 if total %
PROJECTS_PAGE_SIZE else 0)
for prj in available_projects:
fns, sta, err, canceled, pending = dao.get_operation_numbers(
prj.id)
prj.operations_finished = fns
prj.operations_started = sta
prj.operations_error = err
prj.operations_canceled = canceled
prj.operations_pending = pending
prj.disk_size = dao.get_project_disk_size(prj.id)
prj.disk_size_human = format_bytes_human(prj.disk_size)
self.logger.debug("Displaying " + str(len(available_projects)) +
" projects in UI for user " + str(user_id))
return available_projects, pages_no
@staticmethod
def retrieve_all_user_projects(user_id,
page_start=0,
page_size=PROJECTS_PAGE_SIZE):
"""
Return a list with all projects visible for current user, without pagination.
"""
return dao.get_projects_for_user(user_id,
page_start=page_start,
page_size=page_size)
@staticmethod
def get_linkable_projects_for_user(user_id, data_id):
"""
Find projects with are visible for current user, and in which current datatype hasn't been linked yet.
"""
return dao.get_linkable_projects_for_user(user_id, data_id)
@transactional
def remove_project(self, project_id):
"""
Remove Project from DB and File Storage.
"""
try:
project2delete = dao.get_project_by_id(project_id)
self.logger.debug("Deleting project: id=" + str(project_id) +
' name=' + project2delete.name)
project_datatypes = dao.get_datatypes_in_project(project_id)
project_datatypes.sort(key=lambda dt: dt.create_date, reverse=True)
for one_data in project_datatypes:
self.remove_datatype(project_id, one_data.gid, True)
links = dao.get_links_for_project(project_id)
for one_link in links:
dao.remove_entity(Links, one_link.id)
project_bursts = dao.get_bursts_for_project(project_id)
for burst in project_bursts:
dao.remove_entity(burst.__class__, burst.id)
self.storage_interface.remove_project(project2delete)
dao.delete_project(project_id)
self.logger.debug("Deleted project: id=" + str(project_id) +
' name=' + project2delete.name)
except RemoveDataTypeException as excep:
self.logger.exception("Could not execute operation Node Remove!")
raise ProjectServiceException(str(excep))
except FileStructureException as excep:
self.logger.exception("Could not delete because of rights!")
raise ProjectServiceException(str(excep))
except Exception as excep:
self.logger.exception(str(excep))
raise ProjectServiceException(str(excep))
# ----------------- Methods for populating Data-Structure Page ---------------
@staticmethod
def get_datatype_in_group(group):
"""
Return all dataTypes that are the result of the same DTgroup.
"""
return dao.get_datatype_in_group(datatype_group_id=group)
@staticmethod
def get_datatypes_from_datatype_group(datatype_group_id):
"""
Retrieve all dataType which are part from the given dataType group.
"""
return dao.get_datatypes_from_datatype_group(datatype_group_id)
@staticmethod
def load_operation_by_gid(operation_gid):
""" Retrieve loaded Operation from DB"""
return dao.get_operation_by_gid(operation_gid)
@staticmethod
def load_operation_lazy_by_gid(operation_gid):
""" Retrieve lazy Operation from DB"""
return dao.get_operation_lazy_by_gid(operation_gid)
@staticmethod
def get_operation_group_by_id(operation_group_id):
""" Loads OperationGroup from DB"""
return dao.get_operationgroup_by_id(operation_group_id)
@staticmethod
def get_operation_group_by_gid(operation_group_gid):
""" Loads OperationGroup from DB"""
return dao.get_operationgroup_by_gid(operation_group_gid)
@staticmethod
def get_operations_in_group(operation_group):
""" Return all the operations from an operation group. """
return dao.get_operations_in_group(operation_group.id)
@staticmethod
def is_upload_operation(operation_gid):
""" Returns True only if the operation with the given GID is an upload operation. """
return dao.is_upload_operation(operation_gid)
@staticmethod
def get_all_operations_for_uploaders(project_id):
""" Returns all finished upload operations. """
return dao.get_all_operations_for_uploaders(project_id)
def set_operation_and_group_visibility(self,
entity_gid,
is_visible,
is_operation_group=False):
"""
Sets the operation visibility.
If 'is_operation_group' is True than this method will change the visibility for all
the operation from the OperationGroup with the GID field equal to 'entity_gid'.
"""
def set_visibility(op):
# workaround:
# 'reload' the operation so that it has the project property set.
# get_operations_in_group does not eager load it and now we're out of a sqlalchemy session
# write_operation_metadata requires that property
op = dao.get_operation_by_id(op.id)
# end hack
op.visible = is_visible
dao.store_entity(op)
def set_group_descendants_visibility(operation_group_id):
ops_in_group = dao.get_operations_in_group(operation_group_id)
for group_op in ops_in_group:
set_visibility(group_op)
if is_operation_group:
op_group_id = dao.get_operationgroup_by_gid(entity_gid).id
set_group_descendants_visibility(op_group_id)
else:
operation = dao.get_operation_by_gid(entity_gid)
# we assure that if the operation belongs to a group than the visibility will be changed for the entire group
if operation.fk_operation_group is not None:
set_group_descendants_visibility(operation.fk_operation_group)
else:
set_visibility(operation)
def get_operation_details(self, operation_gid, is_group):
"""
:returns: an entity OperationOverlayDetails filled with all information for current operation details.
"""
if is_group:
operation_group = self.get_operation_group_by_gid(operation_gid)
operation = dao.get_operations_in_group(operation_group.id, False,
True)
# Reload, to make sure all attributes lazy are populated as well.
operation = dao.get_operation_by_gid(operation.gid)
no_of_op_in_group = dao.get_operations_in_group(operation_group.id,
is_count=True)
datatype_group = self.get_datatypegroup_by_op_group_id(
operation_group.id)
count_result = dao.count_datatypes_in_group(datatype_group.id)
else:
operation = dao.get_operation_by_gid(operation_gid)
if operation is None:
return None
no_of_op_in_group = 1
count_result = dao.count_resulted_datatypes(operation.id)
user_display_name = dao.get_user_by_id(
operation.fk_launched_by).display_name
burst = dao.get_burst_for_operation_id(operation.id)
datatypes_param, all_special_params = self._review_operation_inputs(
operation.gid)
op_pid = dao.get_operation_process_for_operation(operation.id)
op_details = OperationOverlayDetails(operation, user_display_name,
len(datatypes_param),
count_result, burst,
no_of_op_in_group, op_pid)
# Add all parameter which are set differently by the user on this Operation.
if all_special_params is not None:
op_details.add_scientific_fields(all_special_params)
return op_details
@staticmethod
def get_filterable_meta():
"""
Contains all the attributes by which
the user can structure the tree of DataTypes
"""
return DataTypeMetaData.get_filterable_meta()
def get_project_structure(self, project, visibility_filter, first_level,
second_level, filter_value):
"""
Find all DataTypes (including the linked ones and the groups) relevant for the current project.
In case of a problem, will return an empty list.
"""
metadata_list = []
dt_list = dao.get_data_in_project(project.id, visibility_filter,
filter_value)
for dt in dt_list:
# Prepare the DT results from DB, for usage in controller, by converting into DataTypeMetaData objects
data = {}
is_group = False
group_op = None
# Filter by dt.type, otherwise Links to individual DT inside a group will be mistaken
if dt.type == "DataTypeGroup" and dt.parent_operation.operation_group is not None:
is_group = True
group_op = dt.parent_operation.operation_group
# All these fields are necessary here for dynamic Tree levels.
data[DataTypeMetaData.KEY_DATATYPE_ID] = dt.id
data[DataTypeMetaData.KEY_GID] = dt.gid
data[DataTypeMetaData.KEY_NODE_TYPE] = dt.display_type
data[DataTypeMetaData.KEY_STATE] = dt.state
data[DataTypeMetaData.KEY_SUBJECT] = str(dt.subject)
data[DataTypeMetaData.KEY_TITLE] = dt.display_name
data[DataTypeMetaData.KEY_RELEVANCY] = dt.visible
data[DataTypeMetaData.
KEY_LINK] = dt.parent_operation.fk_launched_in != project.id
data[DataTypeMetaData.
KEY_TAG_1] = dt.user_tag_1 if dt.user_tag_1 else ''
data[DataTypeMetaData.
KEY_TAG_2] = dt.user_tag_2 if dt.user_tag_2 else ''
data[DataTypeMetaData.
KEY_TAG_3] = dt.user_tag_3 if dt.user_tag_3 else ''
data[DataTypeMetaData.
KEY_TAG_4] = dt.user_tag_4 if dt.user_tag_4 else ''
data[DataTypeMetaData.
KEY_TAG_5] = dt.user_tag_5 if dt.user_tag_5 else ''
# Operation related fields:
operation_name = CommonDetails.compute_operation_name(
dt.parent_operation.algorithm.algorithm_category.displayname,
dt.parent_operation.algorithm.displayname)
data[DataTypeMetaData.KEY_OPERATION_TYPE] = operation_name
data[
DataTypeMetaData.
KEY_OPERATION_ALGORITHM] = dt.parent_operation.algorithm.displayname
data[DataTypeMetaData.
KEY_AUTHOR] = dt.parent_operation.user.username
data[
DataTypeMetaData.
KEY_OPERATION_TAG] = group_op.name if is_group else dt.parent_operation.user_group
data[DataTypeMetaData.
KEY_OP_GROUP_ID] = group_op.id if is_group else None
completion_date = dt.parent_operation.completion_date
string_year = completion_date.strftime(
MONTH_YEAR_FORMAT) if completion_date is not None else ""
string_month = completion_date.strftime(
DAY_MONTH_YEAR_FORMAT) if completion_date is not None else ""
data[DataTypeMetaData.KEY_DATE] = date2string(completion_date) if (
completion_date is not None) else ''
data[DataTypeMetaData.KEY_CREATE_DATA_MONTH] = string_year
data[DataTypeMetaData.KEY_CREATE_DATA_DAY] = string_month
data[
DataTypeMetaData.
KEY_BURST] = dt._parent_burst.name if dt._parent_burst is not None else '-None-'
metadata_list.append(DataTypeMetaData(data, dt.invalid))
return StructureNode.metadata2tree(metadata_list, first_level,
second_level, project.id,
project.name)
@staticmethod
def get_datatype_details(datatype_gid):
"""
:returns: an array. First entry in array is an instance of DataTypeOverlayDetails\
The second one contains all the possible states for the specified dataType.
"""
meta_atts = DataTypeOverlayDetails()
states = DataTypeMetaData.STATES
try:
datatype_result = dao.get_datatype_details(datatype_gid)
meta_atts.fill_from_datatype(datatype_result,
datatype_result._parent_burst)
return meta_atts, states, datatype_result
except Exception:
# We ignore exception here (it was logged above, and we want to return no details).
return meta_atts, states, None
def _remove_project_node_files(self,
project_id,
gid,
skip_validation=False):
"""
Delegate removal of a node in the structure of the project.
In case of a problem will THROW StructureException.
"""
try:
project = self.find_project(project_id)
datatype = dao.get_datatype_by_gid(gid)
links = dao.get_links_for_datatype(datatype.id)
op = dao.get_operation_by_id(datatype.fk_from_operation)
if links:
was_link = False
for link in links:
# This means it's only a link and we need to remove it
if link.fk_from_datatype == datatype.id and link.fk_to_project == project.id:
dao.remove_entity(Links, link.id)
was_link = True
if not was_link:
# Create a clone of the operation
# There is no view_model so the view_model_gid is None
new_op = Operation(
op.view_model_gid,
dao.get_system_user().id, links[0].fk_to_project,
datatype.parent_operation.fk_from_algo,
datatype.parent_operation.status,
datatype.parent_operation.start_date,
datatype.parent_operation.completion_date,
datatype.parent_operation.fk_operation_group,
datatype.parent_operation.additional_info,
datatype.parent_operation.user_group,
datatype.parent_operation.range_values)
new_op = dao.store_entity(new_op)
to_project = self.find_project(links[0].fk_to_project)
to_project_path = self.storage_interface.get_project_folder(
to_project.name)
full_path = h5.path_for_stored_index(datatype)
old_folder = self.storage_interface.get_project_folder(
project.name, str(op.id))
vm_full_path = h5.determine_filepath(
op.view_model_gid, old_folder)
self.storage_interface.move_datatype_with_sync(
to_project, to_project_path, new_op.id, full_path,
vm_full_path)
datatype.fk_from_operation = new_op.id
datatype.parent_operation = new_op
dao.store_entity(datatype)
dao.remove_entity(Links, links[0].id)
else:
specific_remover = get_remover(datatype.type)(datatype)
specific_remover.remove_datatype(skip_validation)
except RemoveDataTypeException:
self.logger.exception("Could not execute operation Node Remove!")
raise
except FileStructureException:
self.logger.exception("Remove operation failed")
raise StructureException(
"Remove operation failed for unknown reasons.Please contact system administrator."
)
def remove_operation(self, operation_id):
"""
Remove a given operation
"""
operation = dao.try_get_operation_by_id(operation_id)
if operation is not None:
self.logger.debug("Deleting operation %s " % operation)
datatypes_for_op = dao.get_results_for_operation(operation_id)
for dt in reversed(datatypes_for_op):
self.remove_datatype(operation.project.id, dt.gid, False)
# Here the Operation is mot probably already removed - in case DTs were found inside
# but we still remove it for the case when no DTs exist
dao.remove_entity(Operation, operation.id)
self.storage_interface.remove_operation_data(
operation.project.name, operation_id)
self.storage_interface.push_folder_to_sync(operation.project.name)
self.logger.debug("Finished deleting operation %s " % operation)
else:
self.logger.warning(
"Attempt to delete operation with id=%s which no longer exists."
% operation_id)
def remove_datatype(self, project_id, datatype_gid, skip_validation=False):
"""
Method used for removing a dataType. If the given dataType is a DatatypeGroup
or a dataType from a DataTypeGroup than this method will remove the entire group.
The operation(s) used for creating the dataType(s) will also be removed.
"""
datatype = dao.get_datatype_by_gid(datatype_gid)
if datatype is None:
self.logger.warning(
"Attempt to delete DT[%s] which no longer exists." %
datatype_gid)
return
is_datatype_group = False
datatype_group = None
if dao.is_datatype_group(datatype_gid):
is_datatype_group = True
datatype_group = datatype
elif datatype.fk_datatype_group is not None:
is_datatype_group = True
datatype_group = dao.get_datatype_by_id(datatype.fk_datatype_group)
operations_set = [datatype.fk_from_operation]
correct = True
if is_datatype_group:
operations_set = [datatype_group.fk_from_operation]
self.logger.debug("Removing datatype group %s" % datatype_group)
if datatype_group.fk_parent_burst:
burst = dao.get_generic_entity(BurstConfiguration,
datatype_group.fk_parent_burst,
'gid')[0]
dao.remove_entity(BurstConfiguration, burst.id)
if burst.fk_metric_operation_group:
correct = correct and self._remove_operation_group(
burst.fk_metric_operation_group, project_id,
skip_validation, operations_set)
if burst.fk_operation_group:
correct = correct and self._remove_operation_group(
burst.fk_operation_group, project_id, skip_validation,
operations_set)
else:
self._remove_datatype_group_dts(project_id, datatype_group.id,
skip_validation,
operations_set)
datatype_group = dao.get_datatype_group_by_gid(
datatype_group.gid)
dao.remove_entity(DataTypeGroup, datatype.id)
correct = correct and dao.remove_entity(
OperationGroup, datatype_group.fk_operation_group)
else:
self.logger.debug("Removing datatype %s" % datatype)
self._remove_project_node_files(project_id, datatype.gid,
skip_validation)
# Remove Operation entity in case no other DataType needs them.
project = dao.get_project_by_id(project_id)
for operation_id in operations_set:
dependent_dt = dao.get_generic_entity(DataType, operation_id,
"fk_from_operation")
if len(dependent_dt) > 0:
# Do not remove Operation in case DataType still exist referring it.
continue
op_burst = dao.get_burst_for_operation_id(operation_id)
if op_burst:
correct = correct and dao.remove_entity(
BurstConfiguration, op_burst.id)
correct = correct and dao.remove_entity(Operation, operation_id)
# Make sure Operation folder is removed
self.storage_interface.remove_operation_data(
project.name, operation_id)
self.storage_interface.push_folder_to_sync(project.name)
if not correct:
raise RemoveDataTypeException("Could not remove DataType " +
str(datatype_gid))
def _remove_operation_group(self, operation_group_id, project_id,
skip_validation, operations_set):
metrics_groups = dao.get_generic_entity(DataTypeGroup,
operation_group_id,
'fk_operation_group')
if len(metrics_groups) > 0:
metric_datatype_group_id = metrics_groups[0].id
self._remove_datatype_group_dts(project_id,
metric_datatype_group_id,
skip_validation, operations_set)
dao.remove_entity(DataTypeGroup, metric_datatype_group_id)
return dao.remove_entity(OperationGroup, operation_group_id)
def _remove_datatype_group_dts(self, project_id, dt_group_id,
skip_validation, operations_set):
data_list = dao.get_datatypes_from_datatype_group(dt_group_id)
for adata in data_list:
self._remove_project_node_files(project_id, adata.gid,
skip_validation)
if adata.fk_from_operation not in operations_set:
operations_set.append(adata.fk_from_operation)
def update_metadata(self, submit_data):
"""
Update DataType/ DataTypeGroup metadata
THROW StructureException when input data is invalid.
"""
new_data = dict()
for key in DataTypeOverlayDetails().meta_attributes_list:
if key in submit_data:
value = submit_data[key]
if value == "None":
value = None
if value == "" and key in [
CommonDetails.CODE_OPERATION_TAG,
CommonDetails.CODE_OPERATION_GROUP_ID
]:
value = None
new_data[key] = value
try:
if (CommonDetails.CODE_OPERATION_GROUP_ID in new_data
and new_data[CommonDetails.CODE_OPERATION_GROUP_ID]):
# We need to edit a group
all_data_in_group = dao.get_datatype_in_group(
operation_group_id=new_data[
CommonDetails.CODE_OPERATION_GROUP_ID])
if len(all_data_in_group) < 1:
raise StructureException(
"Inconsistent group, can not be updated!")
# datatype_group = dao.get_generic_entity(DataTypeGroup, all_data_in_group[0].fk_datatype_group)[0]
# all_data_in_group.append(datatype_group)
for datatype in all_data_in_group:
self._edit_data(datatype, new_data, True)
else:
# Get the required DataType and operation from DB to store changes that will be done in XML.
gid = new_data[CommonDetails.CODE_GID]
datatype = dao.get_datatype_by_gid(gid)
self._edit_data(datatype, new_data)
except Exception as excep:
self.logger.exception(excep)
raise StructureException(str(excep))
def _edit_data(self, datatype, new_data, from_group=False):
# type: (DataType, dict, bool) -> None
"""
Private method, used for editing a meta-data XML file and a DataType row
for a given custom DataType entity with new dictionary of data from UI.
"""
# 1. First update Operation fields:
# Update group field if possible
new_group_name = new_data[CommonDetails.CODE_OPERATION_TAG]
empty_group_value = (new_group_name is None or new_group_name == "")
if from_group:
if empty_group_value:
raise StructureException("Empty group is not allowed!")
group = dao.get_generic_entity(
OperationGroup,
new_data[CommonDetails.CODE_OPERATION_GROUP_ID])
if group and len(group) > 0 and new_group_name != group[0].name:
group = group[0]
exists_group = dao.get_generic_entity(OperationGroup,
new_group_name, 'name')
if exists_group:
raise StructureException("Group '" + new_group_name +
"' already exists.")
group.name = new_group_name
dao.store_entity(group)
else:
operation = dao.get_operation_by_id(datatype.fk_from_operation)
operation.user_group = new_group_name
dao.store_entity(operation)
op_folder = self.storage_interface.get_project_folder(
operation.project.name, str(operation.id))
vm_gid = operation.view_model_gid
view_model_file = h5.determine_filepath(vm_gid, op_folder)
if view_model_file:
view_model_class = H5File.determine_type(view_model_file)
view_model = view_model_class()
with ViewModelH5(view_model_file, view_model) as f:
ga = f.load_generic_attributes()
ga.operation_tag = new_group_name
f.store_generic_attributes(ga, False)
else:
self.logger.warning(
"Could not find ViewModel H5 file for op: {}".format(
operation))
# 2. Update GenericAttributes in the associated H5 files:
h5_path = h5.path_for_stored_index(datatype)
with H5File.from_file(h5_path) as f:
ga = f.load_generic_attributes()
ga.subject = new_data[DataTypeOverlayDetails.DATA_SUBJECT]
ga.state = new_data[DataTypeOverlayDetails.DATA_STATE]
ga.operation_tag = new_group_name
if DataTypeOverlayDetails.DATA_TAG_1 in new_data:
ga.user_tag_1 = new_data[DataTypeOverlayDetails.DATA_TAG_1]
if DataTypeOverlayDetails.DATA_TAG_2 in new_data:
ga.user_tag_2 = new_data[DataTypeOverlayDetails.DATA_TAG_2]
if DataTypeOverlayDetails.DATA_TAG_3 in new_data:
ga.user_tag_3 = new_data[DataTypeOverlayDetails.DATA_TAG_3]
if DataTypeOverlayDetails.DATA_TAG_4 in new_data:
ga.user_tag_4 = new_data[DataTypeOverlayDetails.DATA_TAG_4]
if DataTypeOverlayDetails.DATA_TAG_5 in new_data:
ga.user_tag_5 = new_data[DataTypeOverlayDetails.DATA_TAG_5]
f.store_generic_attributes(ga, False)
# 3. Update MetaData in DT Index DB as well.
datatype.fill_from_generic_attributes(ga)
dao.store_entity(datatype)
def _review_operation_inputs(self, operation_gid):
"""
:returns: A list of DataTypes that are used as input parameters for the specified operation.
And a dictionary will all operation parameters different then the default ones.
"""
operation = dao.get_operation_by_gid(operation_gid)
try:
adapter = ABCAdapter.build_adapter(operation.algorithm)
return review_operation_inputs_from_adapter(adapter, operation)
except Exception:
self.logger.exception("Could not load details for operation %s" %
operation_gid)
if operation.view_model_gid:
changed_parameters = dict(
Warning=
"Algorithm changed dramatically. We can not offer more details"
)
else:
changed_parameters = dict(
Warning=
"GID parameter is missing. Old implementation of the operation."
)
return [], changed_parameters
@staticmethod
def get_results_for_operation(operation_id, selected_filter=None):
"""
Retrieve the DataTypes entities resulted after the execution of the given operation.
"""
return dao.get_results_for_operation(operation_id, selected_filter)
@staticmethod
def get_datatype_by_id(datatype_id):
"""Retrieve a DataType DB reference by its id."""
return dao.get_datatype_by_id(datatype_id)
@staticmethod
def get_datatypegroup_by_gid(datatypegroup_gid):
""" Returns the DataTypeGroup with the specified gid. """
return dao.get_datatype_group_by_gid(datatypegroup_gid)
@staticmethod
def get_datatypegroup_by_op_group_id(operation_group_id):
""" Returns the DataTypeGroup with the specified id. """
return dao.get_datatypegroup_by_op_group_id(operation_group_id)
@staticmethod
def get_datatypes_in_project(project_id, only_visible=False):
return dao.get_data_in_project(project_id, only_visible)
@staticmethod
def set_datatype_visibility(datatype_gid, is_visible):
"""
Sets the dataType visibility. If the given dataType is a dataType group or it is part of a
dataType group than this method will set the visibility for each dataType from this group.
"""
def set_visibility(dt):
""" set visibility flag, persist in db and h5"""
dt.visible = is_visible
dt = dao.store_entity(dt)
h5_path = h5.path_for_stored_index(dt)
with H5File.from_file(h5_path) as f:
f.visible.store(is_visible)
def set_group_descendants_visibility(datatype_group_id):
datatypes_in_group = dao.get_datatypes_from_datatype_group(
datatype_group_id)
for group_dt in datatypes_in_group:
set_visibility(group_dt)
datatype = dao.get_datatype_by_gid(datatype_gid)
if isinstance(datatype, DataTypeGroup): # datatype is a group
set_group_descendants_visibility(datatype.id)
datatype.visible = is_visible
dao.store_entity(datatype)
elif datatype.fk_datatype_group is not None: # datatype is member of a group
set_group_descendants_visibility(datatype.fk_datatype_group)
# the datatype to be updated is the parent datatype group
parent = dao.get_datatype_by_id(datatype.fk_datatype_group)
parent.visible = is_visible
dao.store_entity(parent)
else:
# update the single datatype.
set_visibility(datatype)
@staticmethod
def is_datatype_group(datatype_gid):
""" Used to check if the dataType with the specified GID is a DataTypeGroup. """
return dao.is_datatype_group(datatype_gid)
def get_linked_datatypes_storage_path(self, project):
"""
:return: the file paths to the datatypes that are linked in `project`
"""
paths = []
for lnk_dt in dao.get_linked_datatypes_in_project(project.id):
# get datatype as a mapped type
path = h5.path_for_stored_index(lnk_dt)
if path is not None:
paths.append(path)
else:
self.logger.warning(
"Problem when trying to retrieve path on %s:%s!" %
(lnk_dt.type, lnk_dt.gid))
return paths
class DiagnoseDiskUsage(object):
FORMAT_DT = ' {:14} {:20} {:>12} {:>12} {:>12} {:>12}'
HEADER_DT = FORMAT_DT.format('', '', 'disk_size(kib)', 'db_size(kib)',
'delta(kib)', 'ratio(%)')
def __init__(self, prj_id):
self.storage_interface = StorageInterface()
self.expected_files = set()
self.prj_disk_size, self.prj_db_size = 0, 0
try:
dao.session.open_session()
# We do not fetch the project using dao because dao will detach it from the session.
# We want to query on the fly on attribute access and this requires attached objects.
# This code is doing a tree traversal of the db.
# The query on attribute access style fits better than aggregating queries.
self.project = dao.session.query(Project).filter(
Project.id == prj_id).one()
self.expected_files.add(
self.storage_interface.get_project_meta_file_path(
self.project.name))
root_path = self.storage_interface.get_project_folder(
self.project.name)
print()
print('Reporting disk for project {} in {}'.format(
self.project.name, root_path))
print()
print(self.HEADER_DT)
for op in self.project.OPERATIONS:
self.analyse_operation(op)
print(self.HEADER_DT)
self.print_usage_line('Project', 'total', self.prj_disk_size,
self.prj_db_size)
print()
self.list_unexpected_project_files(root_path)
print()
finally:
dao.session.close_session()
@staticmethod
def get_h5_by_gid(root, gid):
for f in os.listdir(root):
fp = os.path.join(root, f)
if gid in f and os.path.isfile(fp):
return fp
@staticmethod
def get_file_kib_size(fp):
return int(round((os.path.getsize(fp) / 1024.)))
@staticmethod
def print_usage_line(col1, col2, actual, expected):
if expected != 0:
ratio = int(100.0 * actual / expected)
if ratio > 200:
ratio = "! %s" % ratio
else:
ratio = str(ratio)
else:
ratio = 'inf'
delta = actual - expected
if delta > 100:
delta = "! %s" % delta
else:
delta = str(delta)
print(
DiagnoseDiskUsage.FORMAT_DT.format(col1, col2,
'{:,}'.format(actual),
'{:,}'.format(expected), delta,
ratio))
def analyse_operation(self, op):
op_disk_size, op_db_size = 0, 0
print('Operation {} : {}'.format(op.id, op.algorithm.name))
for dt in op.DATA_TYPES:
if dt.type == 'DataTypeGroup':
# these have no h5
continue
op_pth = self.storage_interface.get_project_folder(
self.project.name, str(op.id))
dt_pth = self.get_h5_by_gid(op_pth, dt.gid)
dt_actual_disk_size = self.get_file_kib_size(dt_pth)
db_disk_size = dt.disk_size or 0
op_disk_size += dt_actual_disk_size
op_db_size += db_disk_size
self.print_usage_line(dt.gid[:12], dt.type, dt_actual_disk_size,
db_disk_size)
self.expected_files.add(dt_pth)
self.prj_disk_size += op_disk_size
self.prj_db_size += op_db_size
self.print_usage_line('', 'total :', op_disk_size, op_db_size)
print()
def list_unexpected_project_files(self, root_path):
unexpected = []
for r, d, files in os.walk(root_path):
for f in files:
pth = os.path.join(r, f)
if pth not in self.expected_files:
unexpected.append(pth)
print('Unexpected project files :')
for f in unexpected:
print(f)
if not unexpected:
print('yey! none found')
class BurstService(object):
LAUNCH_NEW = 'new'
LAUNCH_BRANCH = 'branch'
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.storage_interface = StorageInterface()
def mark_burst_finished(self,
burst_entity,
burst_status=None,
error_message=None,
store_h5_file=True):
"""
Mark Burst status field.
Also compute 'weight' for current burst: no of operations inside, estimate time on disk...
:param burst_entity: BurstConfiguration to be updated, at finish time.
:param burst_status: BurstConfiguration status. By default BURST_FINISHED
:param error_message: If given, set the status to error and perpetuate the message.
"""
if burst_status is None:
burst_status = BurstConfiguration.BURST_FINISHED
if error_message is not None:
burst_status = BurstConfiguration.BURST_ERROR
try:
# If there are any DataType Groups in current Burst, update their counter.
burst_dt_groups = dao.get_generic_entity(DataTypeGroup,
burst_entity.gid,
"fk_parent_burst")
for dt_group in burst_dt_groups:
dt_group.count_results = dao.count_datatypes_in_group(
dt_group.id)
dt_group.disk_size, dt_group.subject = dao.get_summary_for_group(
dt_group.id)
dao.store_entity(dt_group)
# Update actual Burst entity fields
burst_entity.datatypes_number = dao.count_datatypes_in_burst(
burst_entity.gid)
burst_entity.status = burst_status
burst_entity.error_message = error_message
burst_entity.finish_time = datetime.now()
dao.store_entity(burst_entity)
if store_h5_file:
self.store_burst_configuration(burst_entity)
except Exception:
self.logger.exception(
"Could not correctly update Burst status and meta-data!")
burst_entity.status = burst_status
burst_entity.error_message = "Error when updating Burst Status"
burst_entity.finish_time = datetime.now()
dao.store_entity(burst_entity)
if store_h5_file:
self.store_burst_configuration(burst_entity)
def persist_operation_state(self,
operation,
operation_status,
message=None):
"""
Update Operation instance state. Store it in DB and on HDD/
:param operation: Operation instance
:param operation_status: new status
:param message: message in case of error
:return: operation instance changed
"""
operation.mark_complete(operation_status, message)
operation.queue_full = False
operation = dao.store_entity(operation)
# update burst also
burst_config = self.get_burst_for_operation_id(operation.id)
if burst_config is not None:
burst_status = STATUS_FOR_OPERATION.get(operation_status)
self.mark_burst_finished(burst_config, burst_status, message)
return operation
@staticmethod
def get_burst_for_operation_id(operation_id, is_group=False):
return dao.get_burst_for_operation_id(operation_id, is_group)
def rename_burst(self, burst_id, new_name):
"""
Rename the burst given by burst_id, setting it's new name to
burst_name.
"""
burst = dao.get_burst_by_id(burst_id)
burst.name = new_name
dao.store_entity(burst)
self.store_burst_configuration(burst)
@staticmethod
def get_available_bursts(project_id):
"""
Return all the burst for the current project.
"""
bursts = dao.get_bursts_for_project(
project_id, page_size=MAX_BURSTS_DISPLAYED) or []
return bursts
@staticmethod
def populate_burst_disk_usage(bursts):
"""
Adds a disk_usage field to each burst object.
The disk usage is computed as the sum of the datatypes generated by a burst
"""
sizes = dao.compute_bursts_disk_size([b.gid for b in bursts])
for b in bursts:
b.disk_size = format_bytes_human(sizes[b.gid])
def update_history_status(self, id_list):
"""
For each burst_id received in the id_list read new status from DB and return a list
[id, new_status, is_group, message, running_time] tuple.
"""
result = []
for b_id in id_list:
burst = dao.get_burst_by_id(b_id)
if burst is not None:
if burst.status == burst.BURST_RUNNING:
running_time = datetime.now() - burst.start_time
else:
running_time = burst.finish_time - burst.start_time
running_time = format_timedelta(running_time,
most_significant2=False)
if burst.status == burst.BURST_ERROR:
msg = 'Check Operations page for error Message'
else:
msg = ''
result.append([
burst.id, burst.status, burst.is_group, msg, running_time
])
else:
self.logger.debug("Could not find burst with id=" + str(b_id) +
". Might have been deleted by user!!")
return result
@staticmethod
def update_simulation_fields(burst, op_simulation_id, simulation_gid):
burst.fk_simulation = op_simulation_id
burst.simulator_gid = simulation_gid.hex
burst = dao.store_entity(burst)
return burst
@staticmethod
def load_burst_configuration(burst_config_id):
# type: (int) -> BurstConfiguration
burst_config = dao.get_burst_by_id(burst_config_id)
return burst_config
@staticmethod
def remove_burst_configuration(burst_config_id):
# type: (int) -> None
dao.remove_entity(BurstConfiguration, burst_config_id)
@staticmethod
def prepare_burst_for_pse(burst_config):
# type: (BurstConfiguration) -> (BurstConfiguration)
operation_group = OperationGroup(burst_config.fk_project,
ranges=burst_config.ranges)
operation_group = dao.store_entity(operation_group)
metric_operation_group = OperationGroup(burst_config.fk_project,
ranges=burst_config.ranges)
metric_operation_group = dao.store_entity(metric_operation_group)
burst_config.operation_group = operation_group
burst_config.fk_operation_group = operation_group.id
burst_config.metric_operation_group = metric_operation_group
burst_config.fk_metric_operation_group = metric_operation_group.id
return dao.store_entity(burst_config)
@staticmethod
def store_burst_configuration(burst_config):
project = dao.get_project_by_id(burst_config.fk_project)
bc_path = h5.path_for(burst_config.fk_simulation, BurstConfigurationH5,
burst_config.gid, project.name)
with BurstConfigurationH5(bc_path) as bc_h5:
bc_h5.store(burst_config)
@staticmethod
def load_burst_configuration_from_folder(simulator_folder, project):
bc_h5_filename = DirLoader(
simulator_folder,
None).find_file_for_has_traits_type(BurstConfiguration)
burst_config = BurstConfiguration(project.id)
with BurstConfigurationH5(
os.path.join(simulator_folder, bc_h5_filename)) as bc_h5:
bc_h5.load_into(burst_config)
return burst_config
@staticmethod
def prepare_simulation_name(burst, project_id):
simulation_number = dao.get_number_of_bursts(project_id) + 1
if burst.name is None:
simulation_name = 'simulation_' + str(simulation_number)
else:
simulation_name = burst.name
return simulation_name, simulation_number
def prepare_indexes_for_simulation_results(self, operation,
result_filenames, burst):
indexes = list()
self.logger.debug(
"Preparing indexes for simulation results in operation {}...".
format(operation.id))
for filename in result_filenames:
try:
self.logger.debug(
"Preparing index for filename: {}".format(filename))
index = h5.index_for_h5_file(filename)()
h5_class = h5.REGISTRY.get_h5file_for_index(type(index))
with h5_class(filename) as index_h5:
index.fill_from_h5(index_h5)
index.fill_from_generic_attributes(
index_h5.load_generic_attributes())
index.fk_parent_burst = burst.gid
index.fk_from_operation = operation.id
if operation.fk_operation_group:
datatype_group = dao.get_datatypegroup_by_op_group_id(
operation.fk_operation_group)
self.logger.debug(
"Found DatatypeGroup with id {} for operation {}".
format(datatype_group.id, operation.id))
index.fk_datatype_group = datatype_group.id
# Update the operation group name
operation_group = dao.get_operationgroup_by_id(
operation.fk_operation_group)
operation_group.fill_operationgroup_name(
"TimeSeriesRegionIndex")
dao.store_entity(operation_group)
self.logger.debug(
"Prepared index {} for file {} in operation {}".format(
index.summary_info, filename, operation.id))
indexes.append(index)
except Exception as e:
self.logger.debug(
"Skip preparing index {} because there was an error.".
format(filename))
self.logger.error(e)
self.logger.debug(
"Prepared {} indexes for results in operation {}...".format(
len(indexes), operation.id))
return indexes
def prepare_index_for_metric_result(self, operation, result_filename,
burst):
self.logger.debug(
"Preparing index for metric result in operation {}...".format(
operation.id))
index = h5.index_for_h5_file(result_filename)()
with DatatypeMeasureH5(result_filename) as dti_h5:
index.gid = dti_h5.gid.load().hex
index.metrics = json.dumps(dti_h5.metrics.load())
index.fk_source_gid = dti_h5.analyzed_datatype.load().hex
index.fk_from_operation = operation.id
index.fk_parent_burst = burst.gid
datatype_group = dao.get_datatypegroup_by_op_group_id(
operation.fk_operation_group)
self.logger.debug(
"Found DatatypeGroup with id {} for operation {}".format(
datatype_group.id, operation.id))
index.fk_datatype_group = datatype_group.id
self.logger.debug(
"Prepared index {} for results in operation {}...".format(
index.summary_info, operation.id))
return index
def _update_pse_burst_status(self, burst_config):
operations_in_group = dao.get_operations_in_group(
burst_config.fk_operation_group)
if burst_config.fk_metric_operation_group:
operations_in_group.extend(
dao.get_operations_in_group(
burst_config.fk_metric_operation_group))
operation_statuses = list()
for operation in operations_in_group:
if not has_finished(operation.status):
self.logger.debug(
'Operation {} in group {} is not finished, burst status will not be updated'
.format(operation.id, operation.fk_operation_group))
return
operation_statuses.append(operation.status)
self.logger.debug(
'All operations in burst {} have finished. Will update burst status'
.format(burst_config.id))
if STATUS_ERROR in operation_statuses:
self.mark_burst_finished(
burst_config, BurstConfiguration.BURST_ERROR,
'Some operations in PSE have finished with errors')
elif STATUS_CANCELED in operation_statuses:
self.mark_burst_finished(burst_config,
BurstConfiguration.BURST_CANCELED)
else:
self.mark_burst_finished(burst_config)
def update_burst_status(self, burst_config):
if burst_config.fk_operation_group:
self._update_pse_burst_status(burst_config)
else:
operation = dao.get_operation_by_id(burst_config.fk_simulation)
message = operation.additional_info
if len(message) == 0:
message = None
self.mark_burst_finished(burst_config,
STATUS_FOR_OPERATION[operation.status],
message)
@staticmethod
def prepare_metrics_operation(operation):
# TODO reuse from OperationService and do not duplicate logic here
parent_burst = dao.get_generic_entity(BurstConfiguration,
operation.fk_operation_group,
'fk_operation_group')[0]
metric_operation_group_id = parent_burst.fk_metric_operation_group
range_values = operation.range_values
metric_algo = dao.get_algorithm_by_module(MEASURE_METRICS_MODULE,
MEASURE_METRICS_CLASS)
metric_operation = Operation(None,
operation.fk_launched_by,
operation.fk_launched_in,
metric_algo.id,
status=STATUS_FINISHED,
op_group_id=metric_operation_group_id,
range_values=range_values)
metric_operation.visible = False
metric_operation = dao.store_entity(metric_operation)
op_dir = StorageInterface().get_project_folder(
operation.project.name, str(metric_operation.id))
return op_dir, metric_operation
@staticmethod
def get_range_param_by_name(param_name, all_range_parameters):
for range_param in all_range_parameters:
if param_name == range_param.name:
return range_param
return None
@staticmethod
def handle_range_params_at_loading(burst_config, all_range_parameters):
param1, param2 = None, None
if burst_config.range1:
param1 = RangeParameter.from_json(burst_config.range1)
param1.fill_from_default(
BurstService.get_range_param_by_name(param1.name,
all_range_parameters))
if burst_config.range2 is not None:
param2 = RangeParameter.from_json(burst_config.range2)
param2.fill_from_default(
BurstService.get_range_param_by_name(
param2.name, all_range_parameters))
return param1, param2
def prepare_data_for_burst_copy(self, burst_config_id, burst_name_format,
project):
burst_config = self.load_burst_configuration(burst_config_id)
burst_config_copy = burst_config.clone()
count = dao.count_bursts_with_name(burst_config.name,
burst_config.fk_project)
burst_config_copy.name = burst_name_format.format(
burst_config.name, count + 1)
storage_path = self.storage_interface.get_project_folder(
project.name, str(burst_config.fk_simulation))
simulator = h5.load_view_model(burst_config.simulator_gid,
storage_path)
simulator.generic_attributes = GenericAttributes()
return simulator, burst_config_copy
@staticmethod
def store_burst(burst_config):
return dao.store_entity(burst_config)
def load_simulation_from_zip(self, zip_file, project):
import_service = ImportService()
simulator_folder = import_service.import_simulator_configuration_zip(
zip_file)
simulator_h5_filename = DirLoader(
simulator_folder,
None).find_file_for_has_traits_type(SimulatorAdapterModel)
simulator_h5_filepath = os.path.join(simulator_folder,
simulator_h5_filename)
simulator = h5.load_view_model_from_file(simulator_h5_filepath)
burst_config = self.load_burst_configuration_from_folder(
simulator_folder, project)
burst_config_copy = burst_config.clone()
simulator.generic_attributes.parent_burst = burst_config_copy.gid
return simulator, burst_config_copy, simulator_folder
class TestHPCSchedulerClient(BaseTestCase):
def setup_method(self):
self.storage_interface = StorageInterface()
self.dir_gid = '123'
self.encryption_handler = self.storage_interface.get_encryption_handler(
self.dir_gid)
self.clean_database()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user)
def _prepare_dummy_files(self, tmpdir):
dummy_file1 = os.path.join(str(tmpdir), 'dummy1.txt')
open(dummy_file1, 'a').close()
dummy_file2 = os.path.join(str(tmpdir), 'dummy2.txt')
open(dummy_file2, 'a').close()
job_inputs = [dummy_file1, dummy_file2]
return job_inputs
def test_encrypt_inputs(self, tmpdir):
job_inputs = self._prepare_dummy_files(tmpdir)
job_encrypted_inputs = self.encryption_handler.encrypt_inputs(
job_inputs)
# Encrypted folder has 2 more files are more then plain folder
assert len(job_encrypted_inputs) == len(job_inputs)
def test_decrypt_results(self, tmpdir):
# Prepare encrypted dir
job_inputs = self._prepare_dummy_files(tmpdir)
self.encryption_handler.encrypt_inputs(job_inputs)
encrypted_dir = self.encryption_handler.get_encrypted_dir()
# Unencrypt data
out_dir = os.path.join(str(tmpdir), 'output')
os.mkdir(out_dir)
self.encryption_handler.decrypt_results_to_dir(out_dir)
list_plain_dir = os.listdir(out_dir)
assert len(list_plain_dir) == len(os.listdir(encrypted_dir))
assert 'dummy1.txt' in list_plain_dir
assert 'dummy2.txt' in list_plain_dir
def test_decrypt_files(self, tmpdir):
# Prepare encrypted dir
job_inputs = self._prepare_dummy_files(tmpdir)
enc_files = self.encryption_handler.encrypt_inputs(job_inputs)
# Unencrypt data
out_dir = os.path.join(str(tmpdir), 'output')
os.mkdir(out_dir)
self.encryption_handler.decrypt_files_to_dir([enc_files[1]], out_dir)
list_plain_dir = os.listdir(out_dir)
assert len(list_plain_dir) == 1
assert os.path.basename(enc_files[0]).replace('.aes',
'') not in list_plain_dir
assert os.path.basename(enc_files[1]).replace('.aes',
'') in list_plain_dir
def test_do_operation_launch(self, simulator_factory, operation_factory,
mocker):
# Prepare encrypted dir
op = operation_factory(test_user=self.test_user,
test_project=self.test_project)
sim_folder, sim_gid = simulator_factory(op=op)
self._do_operation_launch(op, sim_gid, mocker)
def _do_operation_launch(self, op, sim_gid, mocker, is_pse=False):
# Prepare encrypted dir
self.dir_gid = sim_gid
self.encryption_handler = StorageInterface.get_encryption_handler(
self.dir_gid)
job_encrypted_inputs = HPCSchedulerClient()._prepare_input(
op, self.dir_gid)
self.encryption_handler.encrypt_inputs(job_encrypted_inputs)
encrypted_dir = self.encryption_handler.get_encrypted_dir()
mocker.patch('tvb.core.operation_hpc_launcher._request_passfile',
_request_passfile_dummy)
mocker.patch(
'tvb.core.operation_hpc_launcher._update_operation_status',
_update_operation_status)
# Call do_operation_launch similarly to CSCS env
plain_dir = self.storage_interface.get_project_folder(
self.test_project.name, 'plain')
do_operation_launch(self.dir_gid, 1000, is_pse, '', op.id, plain_dir)
assert len(os.listdir(encrypted_dir)) == 7
output_path = os.path.join(encrypted_dir,
HPCSchedulerClient.OUTPUT_FOLDER)
assert os.path.exists(output_path)
expected_files = 2
if is_pse:
expected_files = 3
assert len(os.listdir(output_path)) == expected_files
return output_path
def test_do_operation_launch_pse(self, simulator_factory,
operation_factory, mocker):
op = operation_factory(test_user=self.test_user,
test_project=self.test_project)
sim_folder, sim_gid = simulator_factory(op=op)
self._do_operation_launch(op, sim_gid, mocker, is_pse=True)
def test_prepare_inputs(self, operation_factory, simulator_factory):
op = operation_factory(test_user=self.test_user,
test_project=self.test_project)
sim_folder, sim_gid = simulator_factory(op=op)
hpc_client = HPCSchedulerClient()
input_files = hpc_client._prepare_input(op, sim_gid)
assert len(input_files) == 6
def test_prepare_inputs_with_surface(self, operation_factory,
simulator_factory):
op = operation_factory(test_user=self.test_user,
test_project=self.test_project)
sim_folder, sim_gid = simulator_factory(op=op, with_surface=True)
hpc_client = HPCSchedulerClient()
input_files = hpc_client._prepare_input(op, sim_gid)
assert len(input_files) == 9
def test_prepare_inputs_with_eeg_monitor(self, operation_factory,
simulator_factory,
surface_index_factory,
sensors_index_factory,
region_mapping_index_factory,
connectivity_index_factory):
surface_idx, surface = surface_index_factory(cortical=True)
sensors_idx, sensors = sensors_index_factory()
proj = ProjectionSurfaceEEG(sensors=sensors,
sources=surface,
projection_data=numpy.ones(3))
op = operation_factory()
prj_db_db = h5.store_complete(proj, op.id, op.project.name)
prj_db_db.fk_from_operation = op.id
dao.store_entity(prj_db_db)
connectivity = connectivity_index_factory(76, op)
rm_index = region_mapping_index_factory(conn_gid=connectivity.gid,
surface_gid=surface_idx.gid)
eeg_monitor = EEGViewModel(projection=proj.gid, sensors=sensors.gid)
eeg_monitor.region_mapping = rm_index.gid
sim_folder, sim_gid = simulator_factory(op=op,
monitor=eeg_monitor,
conn_gid=connectivity.gid)
hpc_client = HPCSchedulerClient()
input_files = hpc_client._prepare_input(op, sim_gid)
assert len(input_files) == 11
def test_stage_out_to_operation_folder(self, mocker, operation_factory,
simulator_factory,
pse_burst_configuration_factory):
burst = pse_burst_configuration_factory(self.test_project)
op = operation_factory(test_user=self.test_user,
test_project=self.test_project)
op.fk_operation_group = burst.fk_operation_group
dao.store_entity(op)
sim_folder, sim_gid = simulator_factory(op=op)
burst.simulator_gid = sim_gid.hex
dao.store_entity(burst)
output_path = self._do_operation_launch(op,
sim_gid,
mocker,
is_pse=True)
def _stage_out_dummy(dir, sim_gid):
return [
os.path.join(output_path, enc_file)
for enc_file in os.listdir(output_path)
]
mocker.patch.object(HPCSchedulerClient, '_stage_out_results',
_stage_out_dummy)
sim_results_files, metric_op, metric_file = HPCSchedulerClient.stage_out_to_operation_folder(
None, op, sim_gid)
assert op.id != metric_op.id
assert os.path.exists(metric_file)
assert len(sim_results_files) == 1
assert os.path.exists(sim_results_files[0])
def teardown_method(self):
encrypted_dir = self.encryption_handler.get_encrypted_dir()
passfile = self.encryption_handler.get_password_file()
self.storage_interface.remove_files([encrypted_dir, passfile])
self.clean_database()
class TVBLoader(object):
def __init__(self, registry):
self.storage_interface = StorageInterface()
self.registry = registry
def path_for_stored_index(self, dt_index_instance):
# type: (DataType) -> str
""" Given a Datatype(HasTraitsIndex) instance, build where the corresponding H5 should be or is stored"""
if hasattr(dt_index_instance, 'fk_simulation'):
# In case of BurstConfiguration the operation id is on fk_simulation
op_id = dt_index_instance.fk_simulation
else:
op_id = dt_index_instance.fk_from_operation
operation = dao.get_operation_by_id(op_id)
operation_folder = self.storage_interface.get_project_folder(operation.project.name, str(operation.id))
gid = uuid.UUID(dt_index_instance.gid)
h5_file_class = self.registry.get_h5file_for_index(dt_index_instance.__class__)
fname = self.storage_interface.get_filename(h5_file_class.file_name_base(), gid)
return os.path.join(operation_folder, fname)
def path_for(self, op_id, h5_file_class, gid, project_name, dt_class):
return self.storage_interface.path_for(op_id, h5_file_class, gid, project_name, dt_class)
def path_by_dir(self, base_dir, h5_file_class, gid, dt_class):
return self.storage_interface.path_by_dir(base_dir, h5_file_class, gid, dt_class)
def load_from_index(self, dt_index):
# type: (DataType) -> HasTraits
h5_path = self.path_for_stored_index(dt_index)
h5_file_class = self.registry.get_h5file_for_index(dt_index.__class__)
traits_class = self.registry.get_datatype_for_index(dt_index)
with h5_file_class(h5_path) as f:
result_dt = traits_class()
f.load_into(result_dt)
return result_dt
def load_complete_by_function(self, file_path, load_ht_function):
# type: (str, callable) -> (HasTraits, GenericAttributes)
with H5File.from_file(file_path) as f:
try:
datatype_cls = self.registry.get_datatype_for_h5file(f)
except KeyError:
datatype_cls = f.determine_datatype_from_file()
datatype = datatype_cls()
f.load_into(datatype)
ga = f.load_generic_attributes()
sub_dt_refs = f.gather_references(datatype_cls)
for traited_attr, sub_gid in sub_dt_refs:
if sub_gid is None:
continue
is_monitor = False
if isinstance(sub_gid, list):
sub_gid = sub_gid[0]
is_monitor = True
ref_ht = load_ht_function(sub_gid, traited_attr)
if is_monitor:
ref_ht = [ref_ht]
setattr(datatype, traited_attr.field_name, ref_ht)
return datatype, ga
def load_with_references(self, file_path):
def load_ht_function(sub_gid, traited_attr):
ref_idx = dao.get_datatype_by_gid(sub_gid.hex, load_lazy=False)
ref_ht = self.load_from_index(ref_idx)
return ref_ht
return self.load_complete_by_function(file_path, load_ht_function)
def load_with_links(self, file_path):
def load_ht_function(sub_gid, traited_attr):
# Used traited_attr.default for cases similar to ProjectionMonitor which has obsnoise of type Noise and
# it cannot be instantiated due to abstract methods, while the default is Additive()
ref_ht = traited_attr.default or traited_attr.field_type()
ref_ht.gid = sub_gid
return ref_ht
return self.load_complete_by_function(file_path, load_ht_function)
class TestStimulusCreator(TransactionalTestCase):
def transactional_setup_method(self):
"""
Reset the database before each test.
"""
self.test_user = TestFactory.create_user('Stim_User')
self.test_project = TestFactory.create_project(self.test_user,
"Stim_Project")
self.storage_interface = StorageInterface()
zip_path = os.path.join(os.path.dirname(tvb_data.__file__),
'connectivity', 'connectivity_66.zip')
TestFactory.import_zip_connectivity(self.test_user, self.test_project,
zip_path)
self.connectivity = TestFactory.get_entity(self.test_project,
ConnectivityIndex)
cortex = os.path.join(os.path.dirname(tvb_data.surfaceData.__file__),
'cortex_16384.zip')
self.surface = TestFactory.import_surface_zip(self.test_user,
self.test_project,
cortex, CORTICAL)
def transactional_teardown_method(self):
"""
Remove project folders and clean up database.
"""
self.storage_interface.remove_project_structure(self.test_project.name)
def test_create_stimulus_region(self):
weight_array = numpy.zeros(self.connectivity.number_of_regions)
region_stimulus_creator = RegionStimulusCreator()
region_stimulus_creator.storage_path = self.storage_interface.get_project_folder(
self.test_project.name, "42")
view_model = region_stimulus_creator.get_view_model_class()()
view_model.connectivity = self.connectivity.gid
view_model.weight = weight_array
view_model.temporal = TemporalApplicableEquation()
view_model.temporal.parameters['a'] = 1.0
view_model.temporal.parameters['b'] = 2.0
region_stimulus_index = region_stimulus_creator.launch(view_model)
assert region_stimulus_index.temporal_equation == 'TemporalApplicableEquation'
assert json.loads(region_stimulus_index.temporal_parameters) == {
'a': 1.0,
'b': 2.0
}
assert region_stimulus_index.fk_connectivity_gid == self.connectivity.gid
def test_create_stimulus_region_with_operation(self):
weight_array = numpy.zeros(self.connectivity.number_of_regions)
region_stimulus_creator = RegionStimulusCreator()
view_model = region_stimulus_creator.get_view_model_class()()
view_model.connectivity = self.connectivity.gid
view_model.weight = weight_array
view_model.temporal = TemporalApplicableEquation()
view_model.temporal.parameters['a'] = 1.0
view_model.temporal.parameters['b'] = 2.0
OperationService().fire_operation(region_stimulus_creator,
self.test_user,
self.test_project.id,
view_model=view_model)
region_stimulus_index = TestFactory.get_entity(self.test_project,
StimuliRegionIndex)
assert region_stimulus_index.temporal_equation == 'TemporalApplicableEquation'
assert json.loads(region_stimulus_index.temporal_parameters) == {
'a': 1.0,
'b': 2.0
}
assert region_stimulus_index.fk_connectivity_gid == self.connectivity.gid
def test_create_stimulus_surface(self):
surface_stimulus_creator = SurfaceStimulusCreator()
surface_stimulus_creator.storage_path = self.storage_interface.get_project_folder(
self.test_project.name, "42")
view_model = surface_stimulus_creator.get_view_model_class()()
view_model.surface = self.surface.gid
view_model.focal_points_triangles = numpy.array([1, 2, 3])
view_model.spatial = FiniteSupportEquation()
view_model.spatial_amp = 1.0
view_model.spatial_sigma = 1.0
view_model.spatial_offset = 0.0
view_model.temporal = TemporalApplicableEquation()
view_model.temporal.parameters['a'] = 1.0
view_model.temporal.parameters['b'] = 0.0
surface_stimulus_index = surface_stimulus_creator.launch(view_model)
assert surface_stimulus_index.spatial_equation == 'FiniteSupportEquation'
assert surface_stimulus_index.temporal_equation == 'TemporalApplicableEquation'
assert surface_stimulus_index.fk_surface_gid == self.surface.gid
def test_create_stimulus_surface_with_operation(self):
surface_stimulus_creator = SurfaceStimulusCreator()
view_model = surface_stimulus_creator.get_view_model_class()()
view_model.surface = self.surface.gid
view_model.focal_points_triangles = numpy.array([1, 2, 3])
view_model.spatial = FiniteSupportEquation()
view_model.spatial_amp = 1.0
view_model.spatial_sigma = 1.0
view_model.spatial_offset = 0.0
view_model.temporal = TemporalApplicableEquation()
view_model.temporal.parameters['a'] = 1.0
view_model.temporal.parameters['b'] = 0.0
OperationService().fire_operation(surface_stimulus_creator,
self.test_user,
self.test_project.id,
view_model=view_model)
surface_stimulus_index = TestFactory.get_entity(
self.test_project, StimuliSurfaceIndex)
assert surface_stimulus_index.spatial_equation == 'FiniteSupportEquation'
assert surface_stimulus_index.temporal_equation == 'TemporalApplicableEquation'
assert surface_stimulus_index.fk_surface_gid == self.surface.gid
class TestGIFTISurfaceImporter(BaseTestCase):
"""
Unit-tests for GIFTI Surface importer.
"""
GIFTI_SURFACE_FILE = os.path.join(os.path.dirname(demo_data.__file__),
'sample.cortex.gii')
GIFTI_TIME_SERIES_FILE = os.path.join(os.path.dirname(demo_data.__file__),
'sample.time_series.gii')
WRONG_GII_FILE = os.path.abspath(__file__)
def setup_method(self):
self.test_user = TestFactory.create_user('Gifti_User')
self.test_project = TestFactory.create_project(self.test_user,
"Gifti_Project")
self.storage_interface = StorageInterface()
def teardown_method(self):
"""
Clean-up tests data
"""
self.clean_database()
self.storage_interface.remove_project_structure(self.test_project.name)
def test_import_surface_gifti_data(self, operation_factory):
"""
This method tests import of a surface from GIFTI file.
!!! Important: We changed this test to execute only GIFTI parse
because storing surface it takes too long (~ 9min) since
normals needs to be calculated.
"""
operation_id = operation_factory().id
storage_path = self.storage_interface.get_project_folder(
self.test_project.name, str(operation_id))
parser = GIFTIParser(storage_path, operation_id)
surface = parser.parse(self.GIFTI_SURFACE_FILE)
assert 131342 == len(surface.vertices)
assert 262680 == len(surface.triangles)
def test_import_timeseries_gifti_data(self, operation_factory):
"""
This method tests import of a time series from GIFTI file.
!!! Important: We changed this test to execute only GIFTI parse
because storing surface it takes too long (~ 9min) since
normals needs to be calculated.
"""
operation_id = operation_factory().id
storage_path = self.storage_interface.get_project_folder(
self.test_project.name, str(operation_id))
parser = GIFTIParser(storage_path, operation_id)
time_series = parser.parse(self.GIFTI_TIME_SERIES_FILE)
data_shape = time_series[1]
assert 135 == len(data_shape)
assert 143479 == data_shape[0].dims[0]
def test_import_wrong_gii_file(self):
"""
This method tests import of a file in a wrong format
"""
try:
TestFactory.import_surface_gifti(self.test_user, self.test_project,
self.WRONG_GII_FILE)
raise AssertionError(
"Import should fail in case of a wrong GIFTI format.")
except OperationException:
# Expected exception
pass
class ImportService(object):
"""
Service for importing TVB entities into system.
It supports TVB exported H5 files as input, but it should also handle H5 files
generated outside of TVB, as long as they respect the same structure.
"""
def __init__(self):
self.logger = get_logger(__name__)
self.user_id = None
self.storage_interface = StorageInterface()
self.created_projects = []
self.view_model2adapter = self._populate_view_model2adapter()
def _download_and_unpack_project_zip(self, uploaded, uq_file_name,
temp_folder):
if isinstance(uploaded, (FieldStorage, Part)):
if not uploaded.file:
raise ImportException(
"Please select the archive which contains the project structure."
)
with open(uq_file_name, 'wb') as file_obj:
self.storage_interface.copy_file(uploaded.file, file_obj)
else:
shutil.copy2(uploaded, uq_file_name)
try:
self.storage_interface.unpack_zip(uq_file_name, temp_folder)
except FileStructureException as excep:
self.logger.exception(excep)
raise ImportException(
"Bad ZIP archive provided. A TVB exported project is expected!"
)
@staticmethod
def _compute_unpack_path():
"""
:return: the name of the folder where to expand uploaded zip
"""
now = datetime.now()
date_str = "%d-%d-%d_%d-%d-%d_%d" % (now.year, now.month, now.day,
now.hour, now.minute, now.second,
now.microsecond)
uq_name = "%s-ImportProject" % date_str
return os.path.join(TvbProfile.current.TVB_TEMP_FOLDER, uq_name)
@transactional
def import_project_structure(self, uploaded, user_id):
"""
Execute import operations:
1. check if ZIP or folder
2. find all project nodes
3. for each project node:
- create project
- create all operations and groups
- import all images
- create all dataTypes
"""
self.user_id = user_id
self.created_projects = []
# Now compute the name of the folder where to explode uploaded ZIP file
temp_folder = self._compute_unpack_path()
uq_file_name = temp_folder + ".zip"
try:
self._download_and_unpack_project_zip(uploaded, uq_file_name,
temp_folder)
self._import_project_from_folder(temp_folder)
except Exception as excep:
self.logger.exception(
"Error encountered during import. Deleting projects created during this operation."
)
# Remove project folders created so far.
# Note that using the project service to remove the projects will not work,
# because we do not have support for nested transaction.
# Removing from DB is not necessary because in transactional env a simple exception throw
# will erase everything to be inserted.
for project in self.created_projects:
self.storage_interface.remove_project(project)
raise ImportException(str(excep))
finally:
# Now delete uploaded file and temporary folder where uploaded ZIP was exploded.
self.storage_interface.remove_files([uq_file_name, temp_folder])
def _import_project_from_folder(self, temp_folder):
"""
Process each project from the uploaded pack, to extract names.
"""
temp_project_path = None
for root, _, files in os.walk(temp_folder):
if StorageInterface.TVB_PROJECT_FILE in files:
temp_project_path = root
break
if temp_project_path is not None:
update_manager = ProjectUpdateManager(temp_project_path)
if update_manager.checked_version < 3:
raise ImportException(
'Importing projects with versions older than 3 is not supported in TVB 2! '
'Please import the project in TVB 1.5.8 and then launch the current version of '
'TVB in order to upgrade this project!')
update_manager.run_all_updates()
project = self.__populate_project(temp_project_path)
# Populate the internal list of create projects so far, for cleaning up folders, in case of failure
self.created_projects.append(project)
# Ensure project final folder exists on disk
project_path = self.storage_interface.get_project_folder(
project.name)
shutil.move(
os.path.join(temp_project_path,
StorageInterface.TVB_PROJECT_FILE), project_path)
# Now import project operations with their results
self.import_list_of_operations(project, temp_project_path)
# Import images and move them from temp into target
self._store_imported_images(project, temp_project_path,
project.name)
if StorageInterface.encryption_enabled():
self.storage_interface.remove_project(project, True)
def _load_datatypes_from_operation_folder(self, src_op_path,
operation_entity,
datatype_group):
"""
Loads datatypes from operation folder
:returns: Datatype entities as dict {original_path: Dt instance}
"""
all_datatypes = {}
for file_name in os.listdir(src_op_path):
if self.storage_interface.ends_with_tvb_storage_file_extension(
file_name):
h5_file = os.path.join(src_op_path, file_name)
try:
file_update_manager = FilesUpdateManager()
file_update_manager.upgrade_file(h5_file)
datatype = self.load_datatype_from_file(
h5_file, operation_entity.id, datatype_group,
operation_entity.fk_launched_in)
all_datatypes[h5_file] = datatype
except IncompatibleFileManagerException:
os.remove(h5_file)
self.logger.warning(
"Incompatible H5 file will be ignored: %s" % h5_file)
self.logger.exception("Incompatibility details ...")
return all_datatypes
@staticmethod
def check_import_references(file_path, datatype):
h5_class = H5File.h5_class_from_file(file_path)
reference_list = h5_class(file_path).gather_references()
for _, reference_gid in reference_list:
if not reference_gid:
continue
ref_index = load.load_entity_by_gid(reference_gid)
if ref_index is None:
os.remove(file_path)
dao.remove_entity(datatype.__class__, datatype.id)
raise MissingReferenceException(
'Imported file depends on datatypes that do not exist. Please upload '
'those first!')
def _store_or_link_burst_config(self, burst_config, bc_path, project_id):
bc_already_in_tvb = dao.get_generic_entity(BurstConfiguration,
burst_config.gid, 'gid')
if len(bc_already_in_tvb) == 0:
self.store_datatype(burst_config, bc_path)
return 1
return 0
def store_or_link_datatype(self, datatype, dt_path, project_id):
self.check_import_references(dt_path, datatype)
stored_dt_count = 0
datatype_already_in_tvb = load.load_entity_by_gid(datatype.gid)
if not datatype_already_in_tvb:
self.store_datatype(datatype, dt_path)
stored_dt_count = 1
elif datatype_already_in_tvb.parent_operation.project.id != project_id:
AlgorithmService.create_link(datatype_already_in_tvb.id,
project_id)
if datatype_already_in_tvb.fk_datatype_group:
AlgorithmService.create_link(
datatype_already_in_tvb.fk_datatype_group, project_id)
return stored_dt_count
def _store_imported_datatypes_in_db(self, project, all_datatypes):
# type: (Project, dict) -> int
sorted_dts = sorted(
all_datatypes.items(),
key=lambda dt_item: dt_item[1].create_date or datetime.now())
count = 0
for dt_path, datatype in sorted_dts:
count += self.store_or_link_datatype(datatype, dt_path, project.id)
return count
def _store_imported_images(self, project, temp_project_path, project_name):
"""
Import all images from project
"""
images_root = os.path.join(temp_project_path,
StorageInterface.IMAGES_FOLDER)
target_images_path = self.storage_interface.get_images_folder(
project_name)
for root, _, files in os.walk(images_root):
for metadata_file in files:
if self.storage_interface.ends_with_tvb_file_extension(
metadata_file):
self._import_image(root, metadata_file, project.id,
target_images_path)
@staticmethod
def _populate_view_model2adapter():
if len(VIEW_MODEL2ADAPTER) > 0:
return VIEW_MODEL2ADAPTER
view_model2adapter = {}
algos = dao.get_all_algorithms()
for algo in algos:
adapter = ABCAdapter.build_adapter(algo)
view_model_class = adapter.get_view_model_class()
view_model2adapter[view_model_class] = algo
return view_model2adapter
def _retrieve_operations_in_order(self,
project,
import_path,
importer_operation_id=None):
# type: (Project, str, int) -> list[Operation2ImportData]
retrieved_operations = []
for root, _, files in os.walk(import_path):
if OPERATION_XML in files:
# Previous Operation format for uploading previous versions of projects
operation_file_path = os.path.join(root, OPERATION_XML)
operation, operation_xml_parameters, _ = self.build_operation_from_file(
project, operation_file_path)
operation.import_file = operation_file_path
self.logger.debug("Found operation in old XML format: " +
str(operation))
retrieved_operations.append(
Operation2ImportData(
operation,
root,
info_from_xml=operation_xml_parameters))
else:
# We strive for the new format with ViewModelH5
main_view_model = None
dt_paths = []
all_view_model_files = []
for file in files:
if self.storage_interface.ends_with_tvb_storage_file_extension(
file):
h5_file = os.path.join(root, file)
try:
h5_class = H5File.h5_class_from_file(h5_file)
if h5_class is ViewModelH5:
all_view_model_files.append(h5_file)
if not main_view_model:
view_model = h5.load_view_model_from_file(
h5_file)
if type(
view_model
) in self.view_model2adapter.keys():
main_view_model = view_model
else:
file_update_manager = FilesUpdateManager()
file_update_manager.upgrade_file(h5_file)
dt_paths.append(h5_file)
except Exception:
self.logger.warning(
"Unreadable H5 file will be ignored: %s" %
h5_file)
if main_view_model is not None:
alg = self.view_model2adapter[type(main_view_model)]
op_group_id = None
if main_view_model.operation_group_gid:
op_group = dao.get_operationgroup_by_gid(
main_view_model.operation_group_gid.hex)
if not op_group:
op_group = OperationGroup(
project.id,
ranges=json.loads(main_view_model.ranges),
gid=main_view_model.operation_group_gid.hex)
op_group = dao.store_entity(op_group)
op_group_id = op_group.id
operation = Operation(
main_view_model.gid.hex,
project.fk_admin,
project.id,
alg.id,
status=STATUS_FINISHED,
user_group=main_view_model.generic_attributes.
operation_tag,
start_date=datetime.now(),
completion_date=datetime.now(),
op_group_id=op_group_id,
range_values=main_view_model.range_values)
operation.create_date = main_view_model.create_date
operation.visible = main_view_model.generic_attributes.visible
self.logger.debug(
"Found main ViewModel to create operation for it: " +
str(operation))
retrieved_operations.append(
Operation2ImportData(operation, root, main_view_model,
dt_paths, all_view_model_files))
elif len(dt_paths) > 0:
alg = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE,
TVB_IMPORTER_CLASS)
default_adapter = ABCAdapter.build_adapter(alg)
view_model = default_adapter.get_view_model_class()()
view_model.data_file = dt_paths[0]
vm_path = h5.store_view_model(view_model, root)
all_view_model_files.append(vm_path)
operation = Operation(view_model.gid.hex,
project.fk_admin,
project.id,
alg.id,
status=STATUS_FINISHED,
start_date=datetime.now(),
completion_date=datetime.now())
operation.create_date = datetime.min
self.logger.debug(
"Found no ViewModel in folder, so we default to " +
str(operation))
if importer_operation_id:
operation.id = importer_operation_id
retrieved_operations.append(
Operation2ImportData(operation, root, view_model,
dt_paths, all_view_model_files,
True))
return sorted(retrieved_operations,
key=lambda op_data: op_data.order_field)
def create_view_model(self,
operation_entity,
operation_data,
new_op_folder,
generic_attributes=None,
add_params=None):
view_model = self._get_new_form_view_model(
operation_entity, operation_data.info_from_xml)
if add_params is not None:
for element in add_params:
key_attr = getattr(view_model, element[0])
setattr(key_attr, element[1], element[2])
view_model.range_values = operation_entity.range_values
op_group = dao.get_operationgroup_by_id(
operation_entity.fk_operation_group)
if op_group:
view_model.operation_group_gid = uuid.UUID(op_group.gid)
view_model.ranges = json.dumps(op_group.range_references)
view_model.is_metric_operation = 'DatatypeMeasure' in op_group.name
if generic_attributes is not None:
view_model.generic_attributes = generic_attributes
view_model.generic_attributes.operation_tag = operation_entity.user_group
h5.store_view_model(view_model, new_op_folder)
view_model_disk_size = StorageInterface.compute_recursive_h5_disk_usage(
new_op_folder)
operation_entity.view_model_disk_size = view_model_disk_size
operation_entity.view_model_gid = view_model.gid.hex
dao.store_entity(operation_entity)
return view_model
def import_list_of_operations(self,
project,
import_path,
is_group=False,
importer_operation_id=None):
"""
This method scans provided folder and identify all operations that needs to be imported
"""
all_dts_count = 0
all_stored_dts_count = 0
imported_operations = []
ordered_operations = self._retrieve_operations_in_order(
project, import_path, None if is_group else importer_operation_id)
if is_group and len(ordered_operations) > 0:
first_op = dao.get_operation_by_id(importer_operation_id)
vm_path = h5.determine_filepath(first_op.view_model_gid,
os.path.dirname(import_path))
os.remove(vm_path)
ordered_operations[0].operation.id = importer_operation_id
for operation_data in ordered_operations:
if operation_data.is_old_form:
operation_entity, datatype_group = self.import_operation(
operation_data.operation)
new_op_folder = self.storage_interface.get_project_folder(
project.name, str(operation_entity.id))
try:
operation_datatypes = self._load_datatypes_from_operation_folder(
operation_data.operation_folder, operation_entity,
datatype_group)
# Create and store view_model from operation
self.create_view_model(operation_entity, operation_data,
new_op_folder)
self._store_imported_datatypes_in_db(
project, operation_datatypes)
imported_operations.append(operation_entity)
except MissingReferenceException:
operation_entity.status = STATUS_ERROR
dao.store_entity(operation_entity)
elif operation_data.main_view_model is not None:
operation_data.operation.create_date = datetime.now()
operation_data.operation.start_date = datetime.now()
operation_data.operation.completion_date = datetime.now()
do_merge = False
if importer_operation_id:
do_merge = True
operation_entity = dao.store_entity(operation_data.operation,
merge=do_merge)
dt_group = None
op_group = dao.get_operationgroup_by_id(
operation_entity.fk_operation_group)
if op_group:
dt_group = dao.get_datatypegroup_by_op_group_id(
op_group.id)
if not dt_group:
first_op = dao.get_operations_in_group(
op_group.id, only_first_operation=True)
dt_group = DataTypeGroup(
op_group,
operation_id=first_op.id,
state=DEFAULTDATASTATE_INTERMEDIATE)
dt_group = dao.store_entity(dt_group)
# Store the DataTypes in db
dts = {}
all_dts_count += len(operation_data.dt_paths)
for dt_path in operation_data.dt_paths:
dt = self.load_datatype_from_file(dt_path,
operation_entity.id,
dt_group, project.id)
if isinstance(dt, BurstConfiguration):
if op_group:
dt.fk_operation_group = op_group.id
all_stored_dts_count += self._store_or_link_burst_config(
dt, dt_path, project.id)
else:
dts[dt_path] = dt
if op_group:
op_group.fill_operationgroup_name(dt.type)
dao.store_entity(op_group)
try:
stored_dts_count = self._store_imported_datatypes_in_db(
project, dts)
all_stored_dts_count += stored_dts_count
if operation_data.main_view_model.is_metric_operation:
self._update_burst_metric(operation_entity)
imported_operations.append(operation_entity)
new_op_folder = self.storage_interface.get_project_folder(
project.name, str(operation_entity.id))
view_model_disk_size = 0
for h5_file in operation_data.all_view_model_files:
view_model_disk_size += StorageInterface.compute_size_on_disk(
h5_file)
shutil.move(h5_file, new_op_folder)
operation_entity.view_model_disk_size = view_model_disk_size
dao.store_entity(operation_entity)
except MissingReferenceException as excep:
self.storage_interface.remove_operation_data(
project.name, operation_entity.id)
operation_entity.fk_operation_group = None
dao.store_entity(operation_entity)
dao.remove_entity(DataTypeGroup, dt_group.id)
raise excep
else:
self.logger.warning(
"Folder %s will be ignored, as we could not find a serialized "
"operation or DTs inside!" %
operation_data.operation_folder)
# We want importer_operation_id to be kept just for the first operation (the first iteration)
if is_group:
importer_operation_id = None
self._update_dt_groups(project.id)
self._update_burst_configurations(project.id)
return imported_operations, all_dts_count, all_stored_dts_count
@staticmethod
def _get_new_form_view_model(operation, xml_parameters):
# type (Operation) -> ViewModel
algo = dao.get_algorithm_by_id(operation.fk_from_algo)
ad = ABCAdapter.build_adapter(algo)
view_model = ad.get_view_model_class()()
if xml_parameters:
declarative_attrs = type(view_model).declarative_attrs
if isinstance(xml_parameters, str):
xml_parameters = json.loads(xml_parameters)
for param in xml_parameters:
new_param_name = param
if param != '' and param[0] == "_":
new_param_name = param[1:]
new_param_name = new_param_name.lower()
if new_param_name in declarative_attrs:
try:
setattr(view_model, new_param_name,
xml_parameters[param])
except (TraitTypeError, TraitAttributeError):
pass
return view_model
def _import_image(self, src_folder, metadata_file, project_id,
target_images_path):
"""
Create and store a image entity.
"""
figure_dict = StorageInterface().read_metadata_from_xml(
os.path.join(src_folder, metadata_file))
actual_figure = os.path.join(
src_folder,
os.path.split(figure_dict['file_path'])[1])
if not os.path.exists(actual_figure):
self.logger.warning("Expected to find image path %s .Skipping" %
actual_figure)
return
figure_dict['fk_user_id'] = self.user_id
figure_dict['fk_project_id'] = project_id
figure_entity = manager_of_class(ResultFigure).new_instance()
figure_entity = figure_entity.from_dict(figure_dict)
stored_entity = dao.store_entity(figure_entity)
# Update image meta-data with the new details after import
figure = dao.load_figure(stored_entity.id)
shutil.move(actual_figure, target_images_path)
self.logger.debug("Store imported figure")
_, meta_data = figure.to_dict()
self.storage_interface.write_image_metadata(figure, meta_data)
def load_datatype_from_file(self,
current_file,
op_id,
datatype_group=None,
current_project_id=None):
# type: (str, int, DataTypeGroup, int) -> HasTraitsIndex
"""
Creates an instance of datatype from storage / H5 file
:returns: DatatypeIndex
"""
self.logger.debug("Loading DataType from file: %s" % current_file)
h5_class = H5File.h5_class_from_file(current_file)
if h5_class is BurstConfigurationH5:
if current_project_id is None:
op_entity = dao.get_operationgroup_by_id(op_id)
current_project_id = op_entity.fk_launched_in
h5_file = BurstConfigurationH5(current_file)
burst = BurstConfiguration(current_project_id)
burst.fk_simulation = op_id
h5_file.load_into(burst)
result = burst
else:
datatype, generic_attributes = h5.load_with_links(current_file)
already_existing_datatype = h5.load_entity_by_gid(datatype.gid)
if datatype_group is not None and already_existing_datatype is not None:
raise DatatypeGroupImportException(
"The datatype group that you are trying to import"
" already exists!")
index_class = h5.REGISTRY.get_index_for_datatype(
datatype.__class__)
datatype_index = index_class()
datatype_index.fill_from_has_traits(datatype)
datatype_index.fill_from_generic_attributes(generic_attributes)
if datatype_group is not None and hasattr(datatype_index, 'fk_source_gid') and \
datatype_index.fk_source_gid is not None:
ts = h5.load_entity_by_gid(datatype_index.fk_source_gid)
if ts is None:
op = dao.get_operations_in_group(
datatype_group.fk_operation_group,
only_first_operation=True)
op.fk_operation_group = None
dao.store_entity(op)
dao.remove_entity(OperationGroup,
datatype_group.fk_operation_group)
dao.remove_entity(DataTypeGroup, datatype_group.id)
raise DatatypeGroupImportException(
"Please import the time series group before importing the"
" datatype measure group!")
# Add all the required attributes
if datatype_group:
datatype_index.fk_datatype_group = datatype_group.id
if len(datatype_group.subject) == 0:
datatype_group.subject = datatype_index.subject
dao.store_entity(datatype_group)
datatype_index.fk_from_operation = op_id
associated_file = h5.path_for_stored_index(datatype_index)
if os.path.exists(associated_file):
datatype_index.disk_size = StorageInterface.compute_size_on_disk(
associated_file)
result = datatype_index
return result
def store_datatype(self, datatype, current_file=None):
"""This method stores data type into DB"""
try:
self.logger.debug("Store datatype: %s with Gid: %s" %
(datatype.__class__.__name__, datatype.gid))
# Now move storage file into correct folder if necessary
if current_file is not None:
final_path = h5.path_for_stored_index(datatype)
if final_path != current_file:
shutil.move(current_file, final_path)
stored_entry = load.load_entity_by_gid(datatype.gid)
if not stored_entry:
stored_entry = dao.store_entity(datatype)
return stored_entry
except MissingDataSetException as e:
self.logger.exception(e)
error_msg = "Datatype %s has missing data and could not be imported properly." % (
datatype, )
raise ImportException(error_msg)
except IntegrityError as excep:
self.logger.exception(excep)
error_msg = "Could not import data with gid: %s. There is already a one with " \
"the same name or gid." % datatype.gid
raise ImportException(error_msg)
def __populate_project(self, project_path):
"""
Create and store a Project entity.
"""
self.logger.debug("Creating project from path: %s" % project_path)
project_dict = self.storage_interface.read_project_metadata(
project_path)
project_entity = manager_of_class(Project).new_instance()
project_entity = project_entity.from_dict(project_dict, self.user_id)
try:
self.logger.debug("Storing imported project")
return dao.store_entity(project_entity)
except IntegrityError as excep:
self.logger.exception(excep)
error_msg = (
"Could not import project: %s with gid: %s. There is already a "
"project with the same name or gid.") % (project_entity.name,
project_entity.gid)
raise ImportException(error_msg)
def build_operation_from_file(self, project, operation_file):
"""
Create Operation entity from metadata file.
"""
operation_dict = StorageInterface().read_metadata_from_xml(
operation_file)
operation_entity = manager_of_class(Operation).new_instance()
return operation_entity.from_dict(operation_dict, dao, self.user_id,
project.gid)
@staticmethod
def import_operation(operation_entity, migration=False):
"""
Store a Operation entity.
"""
do_merge = False
if operation_entity.id:
do_merge = True
operation_entity = dao.store_entity(operation_entity, merge=do_merge)
operation_group_id = operation_entity.fk_operation_group
datatype_group = None
if operation_group_id is not None:
datatype_group = dao.get_datatypegroup_by_op_group_id(
operation_group_id)
if datatype_group is None and migration is False:
# If no dataType group present for current op. group, create it.
operation_group = dao.get_operationgroup_by_id(
operation_group_id)
datatype_group = DataTypeGroup(
operation_group, operation_id=operation_entity.id)
datatype_group.state = UploadAlgorithmCategoryConfig.defaultdatastate
datatype_group = dao.store_entity(datatype_group)
return operation_entity, datatype_group
def import_simulator_configuration_zip(self, zip_file):
# Now compute the name of the folder where to explode uploaded ZIP file
temp_folder = self._compute_unpack_path()
uq_file_name = temp_folder + ".zip"
if isinstance(zip_file, (FieldStorage, Part)):
if not zip_file.file:
raise ServicesBaseException(
"Could not process the given ZIP file...")
with open(uq_file_name, 'wb') as file_obj:
self.storage_interface.copy_file(zip_file.file, file_obj)
else:
shutil.copy2(zip_file, uq_file_name)
try:
self.storage_interface.unpack_zip(uq_file_name, temp_folder)
return temp_folder
except FileStructureException as excep:
raise ServicesBaseException(
"Could not process the given ZIP file..." + str(excep))
@staticmethod
def _update_burst_metric(operation_entity):
burst_config = dao.get_burst_for_operation_id(operation_entity.id)
if burst_config and burst_config.ranges:
if burst_config.fk_metric_operation_group is None:
burst_config.fk_metric_operation_group = operation_entity.fk_operation_group
dao.store_entity(burst_config)
@staticmethod
def _update_dt_groups(project_id):
dt_groups = dao.get_datatypegroup_for_project(project_id)
for dt_group in dt_groups:
dt_group.count_results = dao.count_datatypes_in_group(dt_group.id)
dts_in_group = dao.get_datatypes_from_datatype_group(dt_group.id)
if dts_in_group:
dt_group.fk_parent_burst = dts_in_group[0].fk_parent_burst
dao.store_entity(dt_group)
@staticmethod
def _update_burst_configurations(project_id):
burst_configs = dao.get_bursts_for_project(project_id)
for burst_config in burst_configs:
burst_config.datatypes_number = dao.count_datatypes_in_burst(
burst_config.gid)
dao.store_entity(burst_config)
class ExportManager(object):
"""
This class provides basic methods for exporting data types of projects in different formats.
"""
all_exporters = {} # Dictionary containing all available exporters
logger = get_logger(__name__)
def __init__(self):
# Here we register all available data type exporters
# If new exporters supported, they should be added here
self._register_exporter(TVBExporter())
self._register_exporter(TVBLinkedExporter())
self.storage_interface = StorageInterface()
def _register_exporter(self, exporter):
"""
This method register into an internal format available exporters.
:param exporter: Instance of a data type exporter (extends ABCExporter)
"""
if exporter is not None:
self.all_exporters[exporter.__class__.__name__] = exporter
def get_exporters_for_data(self, data):
"""
Get available exporters for current data type.
:returns: a dictionary with the {exporter_id : label}
"""
if data is None:
raise InvalidExportDataException("Could not detect exporters for null data")
self.logger.debug("Trying to determine exporters valid for %s" % data.type)
results = {}
# No exporter for None data
if data is None:
return results
for exporterId in self.all_exporters.keys():
exporter = self.all_exporters[exporterId]
if exporter.accepts(data):
results[exporterId] = exporter.get_label()
return results
def export_data(self, data, exporter_id, project, user_public_key=None):
"""
Export provided data using given exporter
:param data: data type to be exported
:param exporter_id: identifier of the exporter to be used
:param project: project that contains data to be exported
:param user_public_key: public key file used for encrypting data before exporting
:returns: a tuple with the following elements
1. name of the file to be shown to user
2. full path of the export file (available for download)
3. boolean which specify if file can be deleted after download
"""
if data is None:
raise InvalidExportDataException("Could not export null data. Please select data to be exported")
if exporter_id is None:
raise ExportException("Please select the exporter to be used for this operation")
if exporter_id not in self.all_exporters:
raise ExportException("Provided exporter identifier is not a valid one")
exporter = self.all_exporters[exporter_id]
if user_public_key is not None:
public_key_path, encryption_password = self.storage_interface.prepare_encryption(project.name)
if isinstance(user_public_key, (FieldStorage, Part)):
with open(public_key_path, 'wb') as file_obj:
self.storage_interface.copy_file(user_public_key.file, file_obj)
else:
shutil.copy2(user_public_key, public_key_path)
else:
public_key_path, encryption_password = None, None
if project is None:
raise ExportException("Please provide the project where data files are stored")
# Now we start the real export
if not exporter.accepts(data):
raise InvalidExportDataException("Current data can not be exported by specified exporter")
# Now compute and create folder where to store exported data
# This will imply to generate a folder which is unique for each export
export_data = None
try:
self.logger.debug("Start export of data: %s" % data.type)
export_data = exporter.export(data, project, public_key_path, encryption_password)
except Exception:
pass
return export_data
@staticmethod
def _get_paths_of_linked_datatypes(project):
linked_paths = ProjectService().get_linked_datatypes_storage_path(project)
if not linked_paths:
# do not export an empty operation
return None, None
# Make an import operation which will contain links to other projects
algo = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE, TVB_IMPORTER_CLASS)
op = model_operation.Operation(None, None, project.id, algo.id)
op.project = project
op.algorithm = algo
op.id = 'links-to-external-projects'
op.start_now()
op.mark_complete(model_operation.STATUS_FINISHED)
return linked_paths, op
def export_project(self, project):
"""
Given a project root and the TVB storage_path, create a ZIP
ready for export.
:param project: project object which identifies project to be exported
"""
if project is None:
raise ExportException("Please provide project to be exported")
folders_to_exclude = self._get_op_with_errors(project.id)
linked_paths, op = self._get_paths_of_linked_datatypes(project)
result_path = self.storage_interface.export_project(project, folders_to_exclude, linked_paths, op)
return result_path
@staticmethod
def _get_op_with_errors(project_id):
"""
Get the operation folders with error base name as list.
"""
operations = dao.get_operations_with_error_in_project(project_id)
op_with_errors = []
for op in operations:
op_with_errors.append(op.id)
return op_with_errors
def export_simulator_configuration(self, burst_id):
burst = dao.get_burst_by_id(burst_id)
if burst is None:
raise InvalidExportDataException("Could not find burst with ID " + str(burst_id))
op_folder = self.storage_interface.get_project_folder(burst.project.name, str(burst.fk_simulation))
all_view_model_paths, all_datatype_paths = h5.gather_references_of_view_model(burst.simulator_gid, op_folder)
burst_path = h5.determine_filepath(burst.gid, op_folder)
all_view_model_paths.append(burst_path)
zip_filename = ABCExporter.get_export_file_name(burst, self.storage_interface.TVB_ZIP_FILE_EXTENSION)
result_path = self.storage_interface.export_simulator_configuration(burst, all_view_model_paths,
all_datatype_paths, zip_filename)
return result_path
