class ProjectUpdateManager(UpdateManager):
"""
This goes through all the scripts that are newer than the version number
written in the current project metadata xml, and executes them on the project folder.
"""
def __init__(self, project_path):
self.project_path = project_path
self.storage_interface = StorageInterface()
# This assumes that old project metadata file can be parsed by current version.
self.project_meta = self.storage_interface.read_project_metadata(
project_path)
from_version = int(self.project_meta.get('version', 0))
super(ProjectUpdateManager,
self).__init__(project_update_scripts, from_version,
TvbProfile.current.version.PROJECT_VERSION)
def run_all_updates(self):
"""
Upgrade the project to the latest structure
Go through all update scripts, from project version up to the current_version in the code
"""
super(ProjectUpdateManager,
self).run_all_updates(project_path=self.project_path)
# update project version in metadata
self.project_meta['version'] = self.current_version
self.storage_interface.write_project_metadata_from_dict(
self.project_path, self.project_meta)
def __init__(self):
# Here we register all available data type exporters
# If new exporters supported, they should be added here
self._register_exporter(TVBExporter())
self._register_exporter(TVBLinkedExporter())
self.export_folder = os.path.join(TvbProfile.current.TVB_STORAGE, self.EXPORT_FOLDER_NAME)
self.storage_interface = StorageInterface()
def _import_projects_from_folder(self, temp_folder):
"""
Process each project from the uploaded pack, to extract names.
"""
project_roots = []
for root, _, files in os.walk(temp_folder):
if StorageInterface.TVB_PROJECT_FILE in files:
project_roots.append(root)
for temp_project_path in project_roots:
update_manager = ProjectUpdateManager(temp_project_path)
update_manager.run_all_updates()
project = self.__populate_project(temp_project_path)
# Populate the internal list of create projects so far, for cleaning up folders, in case of failure
self.created_projects.append(project)
# Ensure project final folder exists on disk
project_path = self.storage_interface.get_project_folder(project.name)
shutil.move(os.path.join(temp_project_path, StorageInterface.TVB_PROJECT_FILE), project_path)
# Now import project operations with their results
self.import_project_operations(project, temp_project_path)
# Import images and move them from temp into target
self._store_imported_images(project, temp_project_path, project.name)
if StorageInterface.encryption_enabled():
StorageInterface.sync_folders(project_path)
shutil.rmtree(project_path)
def test_tvb_export_of_simple_datatype_with_encryption(
self, dummy_datatype_index_factory):
"""
Test export of an encrypted data type which has no data stored on file system
"""
datatype = dummy_datatype_index_factory()
storage_interface = StorageInterface()
import_export_encryption_handler = StorageInterface.get_import_export_encryption_handler(
)
import_export_encryption_handler.generate_public_private_key_pair(
TvbProfile.current.TVB_TEMP_FOLDER)
_, file_path, _ = self.export_manager.export_data(
datatype, self.TVB_EXPORTER, self.test_project,
os.path.join(TvbProfile.current.TVB_TEMP_FOLDER,
import_export_encryption_handler.PUBLIC_KEY_NAME))
assert file_path is not None, "Export process should return path to export file"
assert os.path.exists(
file_path), "Could not find export file: %s on disk." % file_path
result = storage_interface.unpack_zip(
file_path, TvbProfile.current.TVB_TEMP_FOLDER)
encrypted_password_path = import_export_encryption_handler.extract_encrypted_password_from_list(
result)
decrypted_file_path = import_export_encryption_handler.decrypt_content(
encrypted_password_path, result,
os.path.join(TvbProfile.current.TVB_TEMP_FOLDER,
import_export_encryption_handler.PRIVATE_KEY_NAME))[0]
original_path = h5.path_for_stored_index(datatype)
self.compare_files(original_path, decrypted_file_path)
def _do_operation_launch(self, op, sim_gid, mocker, is_pse=False):
# Prepare encrypted dir
self.storage_interface = StorageInterface()
self.dir_gid = sim_gid
job_encrypted_inputs = HPCSchedulerClient()._prepare_input(op, sim_gid)
self.storage_interface.encrypt_inputs(sim_gid, job_encrypted_inputs)
encrypted_dir = self.storage_interface.get_encrypted_dir(self.dir_gid)
mocker.patch('tvb.core.operation_hpc_launcher._request_passfile',
_request_passfile_dummy)
mocker.patch(
'tvb.core.operation_hpc_launcher._update_operation_status',
_update_operation_status)
# Call do_operation_launch similarly to CSCS env
plain_dir = self.storage_interface.get_project_folder(
self.test_project.name, 'plain')
do_operation_launch(sim_gid.hex, 1000, is_pse, '', op.id, plain_dir)
assert len(os.listdir(encrypted_dir)) == 7
output_path = os.path.join(encrypted_dir,
HPCSchedulerClient.OUTPUT_FOLDER)
assert os.path.exists(output_path)
expected_files = 2
if is_pse:
expected_files = 3
assert len(os.listdir(output_path)) == expected_files
return output_path
def transactional_setup_method(self):
"""
Reset the database before each test.
"""
self.project_service = ProjectService()
self.storage_interface = StorageInterface()
self.test_user = TestFactory.create_user()
def stage_out_to_operation_folder(working_dir, operation, simulator_gid):
# type: (Storage, Operation, typing.Union[uuid.UUID, str]) -> (list, Operation, str)
encrypted_files = HPCSchedulerClient._stage_out_results(
working_dir, simulator_gid)
simulation_results = list()
metric_encrypted_file = None
metric_vm_encrypted_file = None
for encrypted_file in encrypted_files:
if os.path.basename(encrypted_file).startswith(
DatatypeMeasureH5.file_name_base()):
metric_encrypted_file = encrypted_file
elif os.path.basename(encrypted_file).startswith(
MEASURE_METRICS_MODEL_CLASS):
metric_vm_encrypted_file = encrypted_file
else:
simulation_results.append(encrypted_file)
storage_interface = StorageInterface()
metric_op, metric_file = HPCSchedulerClient._handle_metric_results(
metric_encrypted_file, metric_vm_encrypted_file, operation,
storage_interface, simulator_gid)
project = dao.get_project_by_id(operation.fk_launched_in)
operation_dir = HPCSchedulerClient.storage_interface.get_project_folder(
project.name, str(operation.id))
h5_filenames = storage_interface.decrypt_files_to_dir(
simulator_gid, simulation_results, operation_dir)
storage_interface.cleanup_encryption_handler(simulator_gid)
LOGGER.info("Decrypted h5: {}".format(h5_filenames))
LOGGER.info("Metric op: {}".format(metric_op))
LOGGER.info("Metric file: {}".format(metric_file))
return h5_filenames, metric_op, metric_file
def do_operation_launch(simulator_gid,
available_disk_space,
is_group_launch,
base_url,
operation_id,
plain_dir='/root/plain'):
try:
log.info("Preparing HPC launch for simulation with id={}".format(
simulator_gid))
populate_datatypes_registry()
log.info("Current TVB profile has HPC run=: {}".format(
TvbProfile.current.hpc.IS_HPC_RUN))
storage_interface = StorageInterface()
_request_passfile(simulator_gid, operation_id, base_url,
storage_interface.get_password_file(simulator_gid))
storage_interface.decrypt_results_to_dir(simulator_gid, plain_dir)
log.info("Current wdir is: {}".format(plain_dir))
view_model = h5.load_view_model(simulator_gid, plain_dir)
adapter_instance = HPCSimulatorAdapter(plain_dir, is_group_launch)
_update_operation_status(STATUS_STARTED, simulator_gid, operation_id,
base_url)
adapter_instance._prelaunch(None, view_model, available_disk_space)
_encrypt_results(adapter_instance, storage_interface, simulator_gid)
_update_operation_status(STATUS_FINISHED, simulator_gid, operation_id,
base_url)
except Exception as excep:
log.error("Could not execute operation {}".format(str(sys.argv[1])))
log.exception(excep)
_update_operation_status(STATUS_ERROR, simulator_gid, operation_id,
base_url)
raise excep
def store_view_model(operation, project, view_model):
storage_path = StorageInterface().get_project_folder(
project.name, str(operation.id))
h5.store_view_model(view_model, storage_path)
view_model_size_on_disk = StorageInterface.compute_recursive_h5_disk_usage(
storage_path)
operation.view_model_disk_size = view_model_size_on_disk
dao.store_entity(operation)
def transactional_setup_method(self):
self.test_user = TestFactory.create_user('Rest_User')
self.test_project = TestFactory.create_project(self.test_user, 'Rest_Project', users=[self.test_user.id])
self.operations_resource = GetOperationsInProjectResource()
self.status_resource = GetOperationStatusResource()
self.results_resource = GetOperationResultsResource()
self.launch_resource = LaunchOperationResource()
self.storage_interface = StorageInterface()
def __init__(self, path):
self.bi_hemispheric = False
self.vertices, self.normals, self.triangles = [], [], []
self.hemisphere_mask = []
self._read_vertices = 0
self.storage_interface = StorageInterface()
self._read(path)
def transactional_teardown_method(self):
"""
Clean-up tests data
"""
# Remove EXPORT folder
export_folder = os.path.join(TvbProfile.current.TVB_STORAGE,
StorageInterface.EXPORT_FOLDER_NAME)
StorageInterface.remove_folder(export_folder, True)
def setup_method(self):
self.storage_interface = StorageInterface()
self.dir_gid = '123'
self.encryption_handler = self.storage_interface.get_encryption_handler(
self.dir_gid)
self.clean_database()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user)
def run(self):
"""
Get the required data from the operation queue and launch the operation.
"""
operation_id = self.operation_id
run_params = [TvbProfile.current.PYTHON_INTERPRETER_PATH, '-m', 'tvb.core.operation_async_launcher',
str(operation_id), TvbProfile.CURRENT_PROFILE_NAME]
current_operation = dao.get_operation_by_id(operation_id)
storage_interface = StorageInterface()
project_folder = storage_interface.get_project_folder(current_operation.project.name)
in_usage = storage_interface.is_in_usage(project_folder)
storage_interface.inc_running_op_count(project_folder)
if not in_usage:
storage_interface.sync_folders(project_folder)
# In the exceptional case where the user pressed stop while the Thread startup is done,
# We should no longer launch the operation.
if self.stopped() is False:
env = os.environ.copy()
env['PYTHONPATH'] = os.pathsep.join(sys.path)
# anything that was already in $PYTHONPATH should have been reproduced in sys.path
launched_process = Popen(run_params, stdout=PIPE, stderr=PIPE, env=env)
LOGGER.debug("Storing pid=%s for operation id=%s launched on local machine." % (operation_id,
launched_process.pid))
op_ident = OperationProcessIdentifier(operation_id, pid=launched_process.pid)
dao.store_entity(op_ident)
if self.stopped():
# In the exceptional case where the user pressed stop while the Thread startup is done.
# and stop_operation is concurrently asking about OperationProcessIdentity.
self.stop_pid(launched_process.pid)
subprocess_result = launched_process.communicate()
LOGGER.info("Finished with launch of operation %s" % operation_id)
returned = launched_process.wait()
LOGGER.info("Return code: {}. Stopped: {}".format(returned, self.stopped()))
LOGGER.info("Thread: {}".format(self))
if returned != 0 and not self.stopped():
# Process did not end as expected. (e.g. Segmentation fault)
burst_service = BurstService()
operation = dao.get_operation_by_id(self.operation_id)
LOGGER.error("Operation suffered fatal failure! Exit code: %s Exit message: %s" % (returned,
subprocess_result))
burst_service.persist_operation_state(operation, STATUS_ERROR,
"Operation failed unexpectedly! Please check the log files.")
del launched_process
storage_interface.check_and_delete(project_folder)
# Give back empty spot now that you finished your operation
CURRENT_ACTIVE_THREADS.remove(self)
LOCKS_QUEUE.put(1)
def _adapt_epileptor_simulations():
"""
Previous Simulations on EpileptorWithPermitivity model, should be converted to use the Epileptor model.
As the parameters from the two models are having different ranges and defaults, we do not translate parameters,
we only set the Epileptor as model instead of EpileptorPermittivityCoupling, and leave the model params to defaults.
"""
session = SA_SESSIONMAKER()
epileptor_old = "EpileptorPermittivityCoupling"
epileptor_new = "Epileptor"
param_model = "model"
try:
all_ep_ops = session.query(model.Operation).filter(
model.Operation.parameters.ilike('%"' + epileptor_old +
'"%')).all()
storage_interface = StorageInterface()
all_bursts = dict()
for ep_op in all_ep_ops:
try:
op_params = parse_json_parameters(ep_op.parameters)
if op_params[param_model] != epileptor_old:
LOGGER.debug("Skipping op " + str(op_params[param_model]) +
" -- " + str(ep_op))
continue
LOGGER.debug("Updating " + str(op_params))
op_params[param_model] = epileptor_new
ep_op.parameters = json.dumps(op_params,
cls=MapAsJson.MapAsJsonEncoder)
LOGGER.debug("New params:" + ep_op.parameters)
storage_interface.write_operation_metadata(ep_op)
burst = dao.get_burst_for_operation_id(ep_op.id)
if burst is not None:
LOGGER.debug("Updating burst:" + str(burst))
burst.prepare_after_load()
burst.simulator_configuration[param_model] = {
'value': epileptor_new
}
burst._simulator_configuration = json.dumps(
burst.simulator_configuration,
cls=MapAsJson.MapAsJsonEncoder)
if burst.id not in all_bursts:
all_bursts[burst.id] = burst
except Exception:
LOGGER.exception("Could not process " + str(ep_op))
session.add_all(all_ep_ops)
session.add_all(list(all_bursts.values()))
session.commit()
except Exception:
LOGGER.exception("Could not update Simulation Epileptor Params")
finally:
session.close()
class TVBLinkedExporter(ABCExporter):
"""
"""
def __init__(self):
self.storage_interface = StorageInterface()
def get_supported_types(self):
return [DataType]
def get_label(self):
return "TVB Format with links"
def export(self, data, data_export_folder, project):
"""
Exports data type:
1. If data is a normal data type, simply exports storage file (HDF format)
2. If data is a DataTypeGroup creates a zip with all files for all data types
"""
self.copy_dt_to_export_folder(data, data_export_folder)
export_data_zip_path = self.get_export_data_zip_path(
data, data_export_folder)
return self.export_data_with_references(export_data_zip_path,
data_export_folder)
def get_export_data_zip_path(self, data, data_export_folder):
zip_file_name = self.get_export_file_name(data)
return os.path.join(os.path.dirname(data_export_folder), zip_file_name)
def export_data_with_references(self, export_data_zip_path,
data_export_folder):
self.storage_interface.write_zip_folder(export_data_zip_path,
data_export_folder)
return None, export_data_zip_path, True
def copy_dt_to_export_folder(self, data, data_export_folder):
data_path = h5.path_for_stored_index(data)
with H5File.from_file(data_path) as f:
file_destination = os.path.join(data_export_folder,
os.path.basename(data_path))
if not os.path.exists(file_destination):
self.storage_interface.copy_file(data_path, file_destination)
sub_dt_refs = f.gather_references()
for _, ref_gid in sub_dt_refs:
if ref_gid:
dt = load.load_entity_by_gid(ref_gid)
self.copy_dt_to_export_folder(dt, data_export_folder)
H5File.remove_metadata_param(file_destination, 'parent_burst')
def get_export_file_extension(self, data):
return "zip"
def skip_group_datatypes(self):
return True
def __init__(self):
self.generic_attributes = GenericAttributes()
self.generic_attributes.subject = DataTypeMetaData.DEFAULT_SUBJECT
self.storage_interface = StorageInterface()
# Will be populate with current running operation's identifier
self.operation_id = None
self.user_id = None
self.submitted_form = None
self.log = get_logger(self.__class__.__module__)
def setup_method(self):
"""
Prepare the usage of a different config file for this class only.
"""
StorageInterface.remove_files([TEST_CONFIG_FILE, TestSQLiteProfile.DEFAULT_STORAGE])
self.old_config_file = TvbProfile.current.TVB_CONFIG_FILE
TvbProfile.current.__class__.TVB_CONFIG_FILE = TEST_CONFIG_FILE
TvbProfile._build_profile_class(TvbProfile.CURRENT_PROFILE_NAME)
self.settings_service = SettingsService()
class TVBExporter(ABCExporter):
"""
This exporter simply provides for download data in TVB format
"""
def __init__(self):
self.storage_interface = StorageInterface()
def get_supported_types(self):
return [DataType]
def get_label(self):
return "TVB Format"
def export(self, data, export_folder, project):
"""
Exports data type:
1. If data is a normal data type, simply exports storage file (HDF format)
2. If data is a DataTypeGroup creates a zip with all files for all data types
"""
download_file_name = self.get_export_file_name(data)
if self.is_data_a_group(data):
all_datatypes = self._get_all_data_types_arr(data)
if all_datatypes is None or len(all_datatypes) == 0:
raise ExportException(
"Could not export a data type group with no data")
zip_file = os.path.join(export_folder, download_file_name)
# Create ZIP archive
self.storage_interface.zip_folders(all_datatypes, project.name,
zip_file)
return download_file_name, zip_file, True
else:
data_file = self.copy_dt_to_export_folder(data, export_folder)
return None, data_file, True
def copy_dt_to_export_folder(self, data, data_export_folder):
data_path = h5.path_for_stored_index(data)
file_destination = os.path.join(data_export_folder,
os.path.basename(data_path))
if not os.path.exists(file_destination):
self.storage_interface.copy_file(data_path, file_destination)
H5File.remove_metadata_param(file_destination, 'parent_burst')
return file_destination
def get_export_file_extension(self, data):
if self.is_data_a_group(data):
return "zip"
else:
return "h5"
def editone(self, project_id=None, cancel=False, save=False, delete=False, **data):
"""
Create or change Project. When project_id is empty we create a
new entity, otherwise we are to edit and existent one.
"""
if cherrypy.request.method == 'POST' and cancel:
raise cherrypy.HTTPRedirect('/project')
if cherrypy.request.method == 'POST' and delete:
self._remove_project(project_id)
raise cherrypy.HTTPRedirect('/project/viewall')
current_user = common.get_logged_user()
is_create = False
if project_id is None or not int(project_id):
is_create = True
data["administrator"] = current_user.display_name
admin_username = current_user.username
else:
current_project = self.project_service.find_project(project_id)
if not save:
# Only when we do not have submitted data,
# populate fields with initial values for edit.
data = dict(name=current_project.name, description=current_project.description)
data["administrator"] = current_project.administrator.display_name
admin_username = current_project.administrator.username
self._mark_selected(current_project)
data["project_id"] = project_id
template_specification = dict(mainContent="project/editone", data=data, isCreate=is_create,
title="Create new project" if is_create else "Edit " + data["name"],
editUsersEnabled=(current_user.username == admin_username))
try:
if cherrypy.request.method == 'POST' and save:
data = EditForm().to_python(data)
saved_project = self.project_service.store_project(current_user, is_create, project_id, **data)
if StorageInterface.encryption_enabled() and is_create:
project_folder = StorageInterface().get_project_folder(saved_project.name)
StorageInterface.sync_folders(project_folder)
shutil.rmtree(project_folder)
self._mark_selected(saved_project)
raise cherrypy.HTTPRedirect('/project/viewall')
except formencode.Invalid as excep:
self.logger.debug(str(excep))
template_specification[common.KEY_ERRORS] = excep.unpack_errors()
except ProjectServiceException as excep:
self.logger.debug(str(excep))
common.set_error_message(excep.message)
raise cherrypy.HTTPRedirect('/project/viewall')
all_users, members, pages = self.user_service.get_users_for_project(current_user.username, project_id)
template_specification['usersList'] = all_users
template_specification['usersMembers'] = [m.id for m in members]
template_specification['usersPages'] = pages
template_specification['usersCurrentPage'] = 1
return self.fill_default_attributes(template_specification, 'properties')
def __init__(self, project_path):
self.project_path = project_path
self.storage_interface = StorageInterface()
# This assumes that old project metadata file can be parsed by current version.
self.project_meta = self.storage_interface.read_project_metadata(
project_path)
from_version = int(self.project_meta.get('version', 0))
super(ProjectUpdateManager,
self).__init__(project_update_scripts, from_version,
TvbProfile.current.version.PROJECT_VERSION)
def teardown_method(self):
"""
Reset the database when test is done.
"""
# Delete TEMP folder
StorageInterface.remove_folder(TvbProfile.current.TVB_TEMP_FOLDER)
# Delete folder where data was exported
if self.zip_path:
StorageInterface.remove_folder(os.path.split(self.zip_path)[0])
self.delete_project_folders()
def test_tvb_linked_export_of_simple_datatype_with_encryption(
self, connectivity_index_factory, surface_index_factory,
region_mapping_index_factory):
"""
Test export of an encrypted data type and its linked data types which have no data stored on file system
"""
conn = connectivity_index_factory()
surface_idx, surface = surface_index_factory(cortical=True)
region_mapping_index = region_mapping_index_factory(
conn_gid=conn.gid, surface_gid=surface.gid.hex)
storage_interface = StorageInterface()
import_export_encryption_handler = StorageInterface.get_import_export_encryption_handler(
)
import_export_encryption_handler.generate_public_private_key_pair(
TvbProfile.current.TVB_TEMP_FOLDER)
_, file_path, _ = self.export_manager.export_data(
region_mapping_index, self.TVB_LINKED_EXPORTER, self.test_project,
os.path.join(TvbProfile.current.TVB_TEMP_FOLDER,
import_export_encryption_handler.PUBLIC_KEY_NAME))
assert file_path is not None, "Export process should return path to export file"
assert os.path.exists(
file_path), "Could not find export file;: %s on disk." % file_path
result = storage_interface.unpack_zip(
file_path, TvbProfile.current.TVB_TEMP_FOLDER)
encrypted_password = import_export_encryption_handler.extract_encrypted_password_from_list(
result)
decrypted_file_paths = import_export_encryption_handler.decrypt_content(
encrypted_password, result,
os.path.join(TvbProfile.current.TVB_TEMP_FOLDER,
import_export_encryption_handler.PRIVATE_KEY_NAME))
original_conn_path = h5.path_for_stored_index(conn)
decrypted_conn_path, idx = (decrypted_file_paths[0], 0) if 'Connectivity' in decrypted_file_paths[0] else \
(decrypted_file_paths[1], 1) if 'Connectivity' in decrypted_file_paths[1] else (decrypted_file_paths[2], 2)
self.compare_files(original_conn_path, decrypted_conn_path)
original_surface_path = h5.path_for_stored_index(surface_idx)
del decrypted_file_paths[idx]
decrypted_surface_path, idx = (decrypted_file_paths[0], 0) if 'Surface' in decrypted_file_paths[0] else \
(decrypted_file_paths[1], 1)
self.compare_files(original_surface_path, decrypted_surface_path)
original_rm_path = h5.path_for_stored_index(region_mapping_index)
del decrypted_file_paths[idx]
self.compare_files(original_rm_path, decrypted_file_paths[0])
def import_conn_h5(project_id, h5_path):
project = dao.get_project_by_id(project_id)
TvbProfile.set_profile(TvbProfile.COMMAND_PROFILE)
now = datetime.now()
date_str = "%d-%d-%d_%d-%d-%d_%d" % (now.year, now.month, now.day, now.hour,
now.minute, now.second, now.microsecond)
uq_name = "%s-Connectivity" % date_str
new_path = os.path.join(TvbProfile.current.TVB_TEMP_FOLDER, uq_name)
StorageInterface.copy_file(h5_path, new_path)
importer = ABCAdapter.build_adapter_from_class(TVBImporter)
view_model = importer.get_view_model_class()()
view_model.data_file = new_path
return OperationService().fire_operation(importer, project.administrator, project_id, view_model=view_model)
def delete_project_folders():
"""
This method deletes folders for all projects from TVB folder.
This is done without any check on database. You might get projects in DB but no folder for them on disk.
"""
BaseStorageTestCase.delete_projects_folders()
for folder in [
os.path.join(TvbProfile.current.TVB_STORAGE,
StorageInterface.EXPORT_FOLDER_NAME),
os.path.join(TvbProfile.current.TVB_STORAGE,
StorageInterface.TEMP_FOLDER)
]:
StorageInterface.remove_folder(folder, True)
os.makedirs(folder)
def load_burst_read_only(self, burst_config_id):
try:
burst_config = self.burst_service.load_burst_configuration(
burst_config_id)
storage_path = StorageInterface().get_project_folder(
self.context.project.name, str(burst_config.fk_simulation))
simulator = h5.load_view_model(burst_config.simulator_gid,
storage_path)
last_loaded_form_url = self.get_url_for_final_fragment(
burst_config)
self.context.init_session_at_burst_loading(burst_config, simulator,
last_loaded_form_url)
form = self.prepare_first_fragment()
self.monitors_handler.build_list_of_monitors_from_view_models(
self.context.simulator)
rendering_rules = SimulatorFragmentRenderingRules(
form,
SimulatorWizzardURLs.SET_CONNECTIVITY_URL,
is_simulation_readonly_load=True,
is_first_fragment=True)
return rendering_rules.to_dict()
except Exception:
# Most probably Burst was removed. Delete it from session, so that client
# has a good chance to get a good response on refresh
self.logger.exception("Error loading burst")
self.context.remove_burst_config_from_session()
raise
def test_adapter_huge_memory_requirement(self, test_adapter_factory):
"""
Test that an MemoryException is raised in case adapter cant launch due to lack of memory.
"""
# Prepare adapter
test_adapter_factory(adapter_class=DummyAdapterHugeMemoryRequired)
adapter = TestFactory.create_adapter("tvb.tests.framework.adapters.dummy_adapter3",
"DummyAdapterHugeMemoryRequired")
# Simulate receiving POST data
form = DummyAdapterHugeMemoryRequiredForm()
view_model = form.get_view_model()()
view_model.test = 5
# Prepare operation for launch
operation = Operation(view_model.gid.hex, self.test_user.id, self.test_project.id, adapter.stored_adapter.id,
status=STATUS_STARTED)
operation = dao.store_entity(operation)
# Store ViewModel in H5
parent_folder = StorageInterface().get_project_folder(self.test_project.name, str(operation.id))
h5.store_view_model(view_model, parent_folder)
# Launch operation
with pytest.raises(NoMemoryAvailableException):
OperationService().initiate_prelaunch(operation, adapter)
def test_export_simulator_configuration(self, operation_factory,
connectivity_index_factory):
"""
Test export of a simulator configuration
"""
conn_gid = uuid.UUID(connectivity_index_factory().gid)
operation = operation_factory(is_simulation=True,
store_vm=True,
test_project=self.test_project,
conn_gid=conn_gid)
burst_configuration = BurstConfiguration(self.test_project.id)
burst_configuration.fk_simulation = operation.id
burst_configuration.simulator_gid = operation.view_model_gid
burst_configuration.name = "Test_burst"
burst_configuration = dao.store_entity(burst_configuration)
op_folder = StorageInterface().get_project_folder(
self.test_project.name, str(operation.id))
BurstService().store_burst_configuration(burst_configuration,
op_folder)
export_file = self.export_manager.export_simulator_configuration(
burst_configuration.id)
assert export_file is not None, "Export process should return path to export file"
assert os.path.exists(
export_file
), "Could not find export file: %s on disk." % export_file
assert zipfile.is_zipfile(
export_file), "Generated file is not a valid ZIP file"
def _configure_job(simulator_gid, available_space, is_group_launch,
operation_id):
# type: (str, int, bool, int) -> (dict, list)
bash_entrypoint = os.path.join(
os.environ[HPCSchedulerClient.TVB_BIN_ENV_KEY],
HPCSettings.HPC_LAUNCHER_SH_SCRIPT)
base_url = TvbProfile.current.web.BASE_URL
inputs_in_container = os.path.join(
HPCSchedulerClient.CONTAINER_INPUT_FOLDER,
StorageInterface.get_encryption_handler(
simulator_gid).current_enc_dirname)
# Build job configuration JSON
my_job = {
HPCSettings.UNICORE_EXE_KEY:
os.path.basename(bash_entrypoint),
HPCSettings.UNICORE_ARGS_KEY: [
simulator_gid, available_space, is_group_launch, base_url,
inputs_in_container, HPCSchedulerClient.HOME_FOLDER_MOUNT,
operation_id
],
HPCSettings.UNICORE_RESOURCER_KEY: {
"CPUs": "1"
}
}
if HPCSchedulerClient.CSCS_PROJECT in os.environ:
my_job[HPCSettings.UNICORE_PROJECT_KEY] = os.environ[
HPCSchedulerClient.CSCS_PROJECT]
return my_job, bash_entrypoint
def update(input_file, burst_match_dict=None):
"""
In order to avoid segmentation faults when updating a batch of files just
start every conversion on a different Python process.
:param input_file: the file that needs to be converted to a newer file storage version.
This should be a file that still uses TVB 2.0 storage
"""
if not os.path.isfile(input_file):
raise FileVersioningException("The input path %s received for upgrading from 2 -> 3 is not a "
"valid file on the disk." % input_file)
folder, file_name = os.path.split(input_file)
storage_manager = StorageInterface.get_storage_manager(input_file)
root_metadata = storage_manager.get_metadata()
class_name = root_metadata[DataTypeMetaData.KEY_CLASS_NAME]
if class_name == "LocalConnectivity":
root_metadata[DataTypeMetaData.KEY_MODULE] = "tvb.datatypes.local_connectivity"
storage_manager.set_metadata(root_metadata)
update_localconnectivity_metadata(folder, file_name)
elif class_name == "RegionMapping":
root_metadata[DataTypeMetaData.KEY_MODULE] = "tvb.datatypes.region_mapping"
root_metadata[TvbProfile.current.version.DATA_VERSION_ATTRIBUTE] = TvbProfile.current.version.DATA_VERSION
storage_manager.set_metadata(root_metadata)
