class ProjectService: """ Services layer for Project entities. """ def __init__(self): self.logger = get_logger(__name__) self.structure_helper = FilesHelper() def store_project(self, current_user, is_create, selected_id, **data): """ We want to create/update a project entity. """ #Validate Unique Name new_name = data["name"] if len(new_name) < 1: raise ProjectServiceException("Invalid project name!") projects_no = dao.count_projects_for_name(new_name, selected_id) if projects_no > 0: err = {'name': 'Please choose another name, this one is used!'} raise formencode.Invalid("Duplicate Name Error", {}, None, error_dict=err) started_operations = dao.get_operation_numbers(selected_id)[1] if started_operations > 0: raise ProjectServiceException("A project can not be renamed while operations are still running!") if is_create: current_proj = model.Project(new_name, current_user.id, data["description"]) self.structure_helper.get_project_folder(current_proj) else: try: current_proj = dao.get_project_by_id(selected_id) except Exception, excep: self.logger.exception("An error has occurred!") raise ProjectServiceException(str(excep)) if current_proj.name != new_name: self.structure_helper.rename_project_structure(current_proj.name, new_name) current_proj.name = new_name current_proj.description = data["description"] #Commit to make sure we have a valid ID current_proj.refresh_update_date() self.structure_helper.write_project_metadata(current_proj) current_proj = dao.store_entity(current_proj) #Retrieve, to initialize lazy attributes current_proj = dao.get_project_by_id(current_proj.id) #Update share settings on current Project entity visited_pages = [] prj_admin = current_proj.administrator.username if 'visited_pages' in data and data['visited_pages']: visited_pages = data['visited_pages'].split(',') for page in visited_pages: members = UserService.retrieve_all_users(prj_admin, int(page))[0] members = [m.id for m in members] dao.delete_members_for_project(current_proj.id, members) selected_user_ids = data["users"] dao.add_members_to_project(current_proj.id, selected_user_ids) #Finish operation self.logger.debug("Edit/Save OK for project:" + str(current_proj.id) + ' by user:' + current_user.username) return current_proj
class ProjectUpdateManager(UpdateManager): """ This goes through all the scripts that are newer than the version number written in the current project metadata xml, and executes them on the project folder. """ def __init__(self, project_path): self.project_path = project_path self.files_helper = FilesHelper() # This assumes that old project metadata file can be parsed by current version. self.project_meta = self.files_helper.read_project_metadata(project_path) from_version = int(self.project_meta.get('version', 0)) super(ProjectUpdateManager, self).__init__(project_update_scripts, from_version, TvbProfile.current.version.PROJECT_VERSION) def run_all_updates(self): """ Upgrade the project to the latest structure Go through all update scripts, from project version up to the current_version in the code """ super(ProjectUpdateManager, self).run_all_updates(project_path=self.project_path) # update project version in metadata self.project_meta['version'] = self.current_version self.files_helper.write_project_metadata_from_dict(self.project_path, self.project_meta)
def launch(self, weights, weights_delimiter, tracts, tracts_delimiter, input_data): """ Execute import operations: process the weights and tracts csv files, then use the reference connectivity passed as input_data for the rest of the attributes. :param weights: csv file containing the weights measures :param tracts: csv file containing the tracts measures :param input_data: a reference connectivity with the additional attributes :raises LaunchException: when the number of nodes in CSV files doesn't match the one in the connectivity """ weights_matrix = self._read_csv_file(weights, weights_delimiter) tract_matrix = self._read_csv_file(tracts, tracts_delimiter) FilesHelper.remove_files([weights, tracts]) if weights_matrix.shape[0] != input_data.number_of_regions: raise LaunchException("The csv files define %s nodes but the connectivity you selected as reference " "has only %s nodes." % (weights_matrix.shape[0], input_data.number_of_regions)) result = Connectivity() result.storage_path = self.storage_path result.centres = input_data.centres result.region_labels = input_data.region_labels result.weights = weights_matrix result.tract_lengths = tract_matrix result.orientations = input_data.orientations result.areas = input_data.areas result.cortical = input_data.cortical result.hemispheres = input_data.hemispheres return result
class ImportService(): """ Service for importing TVB entities into system. It supports TVB exported H5 files as input, but it should also handle H5 files generated outside of TVB, as long as they respect the same structure. """ def __init__(self): self.logger = get_logger(__name__) self.user_id = None self.files_helper = FilesHelper() self.created_projects = [] def _download_and_unpack_project_zip(self, uploaded, uq_file_name, temp_folder): if isinstance(uploaded, FieldStorage) or isinstance(uploaded, Part): if not uploaded.file: raise ProjectImportException("Please select the archive which contains the project structure.") with open(uq_file_name, 'wb') as file_obj: self.files_helper.copy_file(uploaded.file, file_obj) else: shutil.copy2(uploaded, uq_file_name) try: self.files_helper.unpack_zip(uq_file_name, temp_folder) except FileStructureException, excep: self.logger.exception(excep) raise ProjectImportException("Bad ZIP archive provided. A TVB exported project is expected!")
def launch(self, weights, tracts, input_data): """ Execute import operations: process the weights and tracts csv files, then use the reference connectivity passed as input_data for the rest of the attributes. :param weights: csv file containing the weights measures :param tracts: csv file containing the tracts measures :param input_data: a reference connectivity with the additional attributes :raises LaunchException: when the number of nodes in CSV files doesn't match the one in the connectivity """ dti_service = DTIPipelineService() dti_service._process_csv_file(weights, dti_service.WEIGHTS_FILE) dti_service._process_csv_file(tracts, dti_service.TRACT_FILE) weights_matrix = read_list_data(os.path.join(os.path.dirname(weights), dti_service.WEIGHTS_FILE)) tract_matrix = read_list_data(os.path.join(os.path.dirname(tracts), dti_service.TRACT_FILE)) FilesHelper.remove_files([os.path.join(os.path.dirname(weights), dti_service.WEIGHTS_FILE), os.path.join(os.path.dirname(tracts), dti_service.TRACT_FILE)]) if weights_matrix.shape[0] != input_data.orientations.shape[0]: raise LaunchException("The csv files define %s nodes but the connectivity you selected as reference " "has only %s nodes." % (weights_matrix.shape[0], input_data.orientations.shape[0])) result = Connectivity() result.storage_path = self.storage_path result.nose_correction = input_data.nose_correction result.centres = input_data.centres result.region_labels = input_data.region_labels result.weights = weights_matrix result.tract_lengths = tract_matrix result.orientations = input_data.orientations result.areas = input_data.areas result.cortical = input_data.cortical result.hemispheres = input_data.hemispheres return result
def export(self, data, export_folder, project): """ Exports data type: 1. If data is a normal data type, simply exports storage file (HDF format) 2. If data is a DataTypeGroup creates a zip with all files for all data types """ download_file_name = self.get_export_file_name(data) files_helper = FilesHelper() if self.is_data_a_group(data): all_datatypes = self._get_all_data_types_arr(data) if all_datatypes is None or len(all_datatypes) == 0: raise ExportException("Could not export a data type group with no data") zip_file = os.path.join(export_folder, download_file_name) # Now process each data type from group and add it to ZIP file operation_folders = [] for data_type in all_datatypes: operation_folder = files_helper.get_operation_folder(project.name, data_type.fk_from_operation) operation_folders.append(operation_folder) # Create ZIP archive files_helper.zip_folders(zip_file, operation_folders, self.OPERATION_FOLDER_PREFIX) return download_file_name, zip_file, True else: project_folder = files_helper.get_project_folder(project) data_file = os.path.join(project_folder, data.get_storage_file_path()) return download_file_name, data_file, False
def export_project(self, project, optimize_size=False): """ Given a project root and the TVB storage_path, create a ZIP ready for export. :param project: project object which identifies project to be exported """ if project is None: raise ExportException("Please provide project to be exported") files_helper = FilesHelper() project_folder = files_helper.get_project_folder(project) project_datatypes = self._gather_project_datatypes(project, optimize_size) to_be_exported_folders = [] considered_op_ids = [] min_dt_date = datetime.now() if optimize_size: ## take only the DataType with visibility flag set ON for dt in project_datatypes: if dt[KEY_OPERATION_ID] not in considered_op_ids: to_be_exported_folders.append({'folder': files_helper.get_project_folder(project, str(dt[KEY_OPERATION_ID])), 'archive_path_prefix': str(dt[KEY_OPERATION_ID]) + os.sep}) considered_op_ids.append(dt[KEY_OPERATION_ID]) if min_dt_date > dt[KEY_DT_DATE]: min_dt_date = dt[KEY_DT_DATE] else: to_be_exported_folders.append({'folder': project_folder, 'archive_path_prefix': '', 'exclude': ["TEMP"]}) if project_datatypes: min_dt_date = min([dt[KEY_DT_DATE] for dt in project_datatypes]) # Compute path and name of the zip file now = datetime.now() date_str = now.strftime("%Y-%m-%d_%H-%M") zip_file_name = "%s_%s.%s" % (date_str, project.name, self.ZIP_FILE_EXTENSION) export_folder = self._build_data_export_folder(project) result_path = os.path.join(export_folder, zip_file_name) with TvbZip(result_path, "w") as zip_file: # Pack project [filtered] content into a ZIP file: LOG.debug("Done preparing, now we will write folders " + str(len(to_be_exported_folders))) LOG.debug(str(to_be_exported_folders)) for pack in to_be_exported_folders: zip_file.write_folder(**pack) LOG.debug("Done exporting files, now we will write the burst configurations...") self._export_bursts(project, project_datatypes, zip_file) LOG.debug("Done exporting burst configurations, now we will export linked DTs") self._export_linked_datatypes(project, zip_file, min_dt_date) ## Make sure the Project.xml file gets copied: if optimize_size: LOG.debug("Done linked, now we write the project xml") zip_file.write(files_helper.get_project_meta_file_path(project.name), files_helper.TVB_PROJECT_FILE) LOG.debug("Done, closing") return result_path
def _update_datatype_disk_size(self, file_path): """ Computes and updates the disk_size attribute of the DataType, for which was created the given file. """ file_handler = FilesHelper() datatype_gid = self._get_manager(file_path).get_gid_attribute() datatype = dao.get_datatype_by_gid(datatype_gid) if datatype is not None: datatype.disk_size = file_handler.compute_size_on_disk(file_path) dao.store_entity(datatype)
def initialize_storage(): """ Create Projects storage root folder in case it does not exist. """ try: helper = FilesHelper() helper.check_created() except FileStructureException: # Do nothing, because we do not have any UI to display exception logger = get_logger("tvb.core.services.initialize_storage") logger.exception("Could not make sure the root folder exists!")
def get_gifty_file_name(project_id, desired_name): """ Compute non-existent file name, in the TEMP folder of the given project. Try desired_name, and if already exists, try adding a number. """ if project_id: project = dao.get_project_by_id(project_id) file_helper = FilesHelper() temp_path = file_helper.get_project_folder(project, FilesHelper.TEMP_FOLDER) return get_unique_file_name(temp_path, desired_name)[0] return get_unique_file_name(cfg.TVB_STORAGE, desired_name)[0]
def _adapt_epileptor_simulations(): """ Previous Simulations on EpileptorWithPermitivity model, should be converted to use the Epileptor model. As the parameters from the two models are having different ranges and defaults, we do not translate parameters, we only set the Epileptor as model instead of EpileptorPermittivityCoupling, and leave the model params to defaults. """ session = SA_SESSIONMAKER() epileptor_old = "EpileptorPermittivityCoupling" epileptor_new = "Epileptor" param_model = "model" try: all_ep_ops = session.query(model.Operation).filter( model.Operation.parameters.ilike('%"' + epileptor_old + '"%')).all() files_helper = FilesHelper() all_bursts = dict() for ep_op in all_ep_ops: try: op_params = parse_json_parameters(ep_op.parameters) if op_params[param_model] != epileptor_old: LOGGER.debug("Skipping op " + str(op_params[param_model]) + " -- " + str(ep_op)) continue LOGGER.debug("Updating " + str(op_params)) op_params[param_model] = epileptor_new ep_op.parameters = json.dumps(op_params, cls=MapAsJson.MapAsJsonEncoder) LOGGER.debug("New params:" + ep_op.parameters) files_helper.write_operation_metadata(ep_op) burst = dao.get_burst_for_operation_id(ep_op.id) if burst is not None: LOGGER.debug("Updating burst:" + str(burst)) burst.prepare_after_load() burst.simulator_configuration[param_model] = {'value': epileptor_new} burst._simulator_configuration = json.dumps(burst.simulator_configuration, cls=MapAsJson.MapAsJsonEncoder) if not all_bursts.has_key(burst.id): all_bursts[burst.id] = burst except Exception: LOGGER.exception("Could not process " + str(ep_op)) session.add_all(all_ep_ops) session.add_all(all_bursts.values()) session.commit() except Exception: LOGGER.exception("Could not update Simulation Epileptor Params") finally: session.close()
def _build_data_export_folder(self, data): """ This method computes the folder where results of an export operation will be stored for a while (e.g until download is done; or for 1 day) """ now = datetime.now() date_str = "%d-%d-%d_%d-%d-%d_%d" % (now.year, now.month, now.day, now.hour, now.minute, now.second, now.microsecond) tmp_str = date_str + "@" + data.gid data_export_folder = os.path.join(self.export_folder, tmp_str) files_helper = FilesHelper() files_helper.check_created(data_export_folder) return data_export_folder
def introduce_unmapped_node(out_pth, conn_zip_pth): """ Creates a connectivity with one extra node in the first position. This node represents the unmapped regions. :param out_pth: destination path :param conn_zip_pth: connectivity zip path. """ fh = FilesHelper() tmp_pth = os.path.splitext(out_pth)[0] fh.check_created(tmp_pth) files = fh.unpack_zip(conn_zip_pth, tmp_pth) for file_name in files: file_name_low = file_name.lower() if "centres" in file_name_low: with open(file_name) as f: lines = f.readlines() with open(file_name, "w") as f: f.write("None 0.000000 0.000000 0.000000\n") f.writelines(lines) elif "weight" in file_name_low or "tract" in file_name_low: with open(file_name) as f: lines = f.readlines() nr_regions = len(lines) with open(file_name, "w") as f: f.write(" 0.0000000e+00" * (nr_regions + 1) + "\n") for line in lines: f.write(" 0.0000000e+00" + line) else: raise Exception("this transformation does not support the file " + file_name) fh.zip_folder(out_pth, tmp_pth) fh.remove_folder(tmp_pth)
def __init__(self): micro_postfix = "_%d" % int(time.time() * 1000000) # Here create all structures needed later for data types creation self.files_helper = FilesHelper() # First create user user = model.User("datatype_factory_user" + micro_postfix, "test_pass", "*****@*****.**" + micro_postfix, True, "user") self.user = dao.store_entity(user) # Now create a project project_service = ProjectService() data = dict(name='DatatypesFactoryProject' + micro_postfix, description='test_desc', users=[]) self.project = project_service.store_project(self.user, True, None, **data) # Create algorithm alg_category = model.AlgorithmCategory('one', True) dao.store_entity(alg_category) ad = model.Algorithm(SIMULATOR_MODULE, SIMULATOR_CLASS, alg_category.id) self.algorithm = dao.get_algorithm_by_module(SIMULATOR_MODULE, SIMULATOR_CLASS) if self.algorithm is None: self.algorithm = dao.store_entity(ad) # Create an operation self.meta = {DataTypeMetaData.KEY_SUBJECT: self.USER_FULL_NAME, DataTypeMetaData.KEY_STATE: self.DATATYPE_STATE} operation = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters', meta=json.dumps(self.meta), status=model.STATUS_FINISHED) self.operation = dao.store_entity(operation)
def _store_imported_datatypes_in_db(self, project, all_datatypes, dt_burst_mappings, burst_ids_mapping): def by_time(dt): return dt.create_date or datetime.now() if burst_ids_mapping is None: burst_ids_mapping = {} if dt_burst_mappings is None: dt_burst_mappings = {} all_datatypes.sort(key=by_time) for datatype in all_datatypes: old_burst_id = dt_burst_mappings.get(datatype.gid) if old_burst_id is not None: datatype.fk_parent_burst = burst_ids_mapping[old_burst_id] datatype_allready_in_tvb = dao.get_datatype_by_gid(datatype.gid) if not datatype_allready_in_tvb: # Compute disk size. Similar to ABCAdapter._capture_operation_results. # No need to close the h5 as we have not written to it. associated_file = os.path.join(datatype.storage_path, datatype.get_storage_file_name()) datatype.disk_size = FilesHelper.compute_size_on_disk(associated_file) self.store_datatype(datatype) else: FlowService.create_link([datatype_allready_in_tvb.id], project.id)
def setUp(self): """ Set up the context needed by the tests. """ self.files_helper = FilesHelper() self.test_user = TestFactory.create_user() self.test_project = TestFactory.create_project(self.test_user, self.PROJECT_NAME)
def __init__(self): micro_postfix = "_%d" % int(time.time() * 1000000) # Here create all structures needed later for data types creation self.files_helper = FilesHelper() # First create user user = model.User("datatype_factory_user" + micro_postfix, "test_pass", "*****@*****.**" + micro_postfix, True, "user") self.user = dao.store_entity(user) # Now create a project project_service = ProjectService() data = dict(name='DatatypesFactoryProject' + micro_postfix, description='test_desc', users=[]) self.project = project_service.store_project(self.user, True, None, **data) # Create algorithm alg_category = model.AlgorithmCategory('one', True) dao.store_entity(alg_category) alg_group = model.AlgorithmGroup("test_module1", "classname1", alg_category.id) dao.store_entity(alg_group) algorithm = model.Algorithm(alg_group.id, 'id', name='', req_data='', param_name='', output='') self.algorithm = dao.store_entity(algorithm) #Create an operation self.meta = {DataTypeMetaData.KEY_SUBJECT: self.USER_FULL_NAME, DataTypeMetaData.KEY_STATE: self.DATATYPE_STATE} operation = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters', meta=json.dumps(self.meta), status=model.STATUS_FINISHED, method_name=ABCAdapter.LAUNCH_METHOD) self.operation = dao.store_entity(operation)
def transactional_setup_method(self): """ Reset the database before each test. """ self.project_service = ProjectService() self.structure_helper = FilesHelper() self.test_user = TestFactory.create_user()
def export_project(self, project): """ Given a project root and the TVB storage_path, create a ZIP ready for export. :param project: project object which identifies project to be exported """ if project is None: raise ExportException("Please provide project to be exported") files_helper = FilesHelper() project_folder = files_helper.get_project_folder(project) bursts_dict = {} datatype_burst_mapping = {} bursts_count = dao.get_bursts_for_project(project.id, count=True) for start_idx in range(0, bursts_count, BURST_PAGE_SIZE): bursts = dao.get_bursts_for_project(project.id, page_start=start_idx, page_end=start_idx + BURST_PAGE_SIZE) for burst in bursts: self._build_burst_export_dict(burst, bursts_dict) datatypes_count = dao.get_datatypes_for_project(project.id, count=True) for start_idx in range(0, datatypes_count, DATAYPES_PAGE_SIZE): datatypes = dao.get_datatypes_for_project(project.id, page_start=start_idx, page_end=start_idx + DATAYPES_PAGE_SIZE) for datatype in datatypes: datatype_burst_mapping[datatype.gid] = datatype.fk_parent_burst # Compute path and name of the zip file now = datetime.now() date_str = now.strftime("%Y-%m-%d_%H-%M") zip_file_name = "%s_%s.%s" % (date_str, project.name, self.ZIP_FILE_EXTENSION) export_folder = self._build_data_export_folder(project) result_path = os.path.join(export_folder, zip_file_name) bursts_file_name = os.path.join(project_folder, BURST_INFO_FILE) burst_info = {BURSTS_DICT_KEY: bursts_dict, DT_BURST_MAP: datatype_burst_mapping} with open(bursts_file_name, 'w') as bursts_file: bursts_file.write(json.dumps(burst_info)) # pack project content into a ZIP file result_zip = files_helper.zip_folder(result_path, project_folder) # remove these files, since we only want them in export archive os.remove(bursts_file_name) return result_zip
def setUp(self): """ Reset the database before each test. """ config.EVENTS_FOLDER = '' self.project_service = ProjectService() self.structure_helper = FilesHelper() self.test_user = TestFactory.create_user()
def _export_linked_datatypes(self, project, zip_file): files_helper = FilesHelper() linked_paths = self._get_linked_datatypes_storage_path(project) if not linked_paths: # do not export an empty operation return # Make a import operation which will contain links to other projects algo = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE, TVB_IMPORTER_CLASS) op = model.Operation(None, project.id, algo.id, '') op.project = project op.algorithm = algo op.id = 'links-to-external-projects' op.start_now() op.mark_complete(model.STATUS_FINISHED) # write operation.xml to disk files_helper.write_operation_metadata(op) op_folder = files_helper.get_operation_folder(op.project.name, op.id) operation_xml = files_helper.get_operation_meta_file_path(op.project.name, op.id) op_folder_name = os.path.basename(op_folder) # add operation.xml zip_file.write(operation_xml, op_folder_name + '/' + os.path.basename(operation_xml)) # add linked datatypes to archive in the import operation for pth in linked_paths: zip_pth = op_folder_name + '/' + os.path.basename(pth) zip_file.write(pth, zip_pth) # remove these files, since we only want them in export archive files_helper.remove_folder(op_folder)
def __init__(self, project_path): self.project_path = project_path self.files_helper = FilesHelper() # This assumes that old project metadata file can be parsed by current version. self.project_meta = self.files_helper.read_project_metadata(project_path) from_version = int(self.project_meta.get('version', 0)) super(ProjectUpdateManager, self).__init__(project_update_scripts, from_version, TvbProfile.current.version.PROJECT_VERSION)
def __init__(self): # It will be populate with key from DataTypeMetaData self.meta_data = {DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT} self.file_handler = FilesHelper() self.storage_path = '.' # Will be populate with current running operation's identifier self.operation_id = None self.user_id = None self.log = get_logger(self.__class__.__module__) self.tree_manager = InputTreeManager()
def search_and_export_ts(project_id, export_folder=os.path.join("~", "TVB")): #### This is the simplest filter you could write: filter and entity by Subject filter_connectivity = FilterChain(fields=[FilterChain.datatype + '.subject'], operations=["=="], values=[DataTypeMetaData.DEFAULT_SUBJECT]) connectivities = _retrieve_entities_by_filters(Connectivity, project_id, filter_connectivity) #### A more complex filter: by linked entity (connectivity), BOLD monitor, sampling, operation param: filter_timeseries = FilterChain(fields=[FilterChain.datatype + '._connectivity', FilterChain.datatype + '._title', FilterChain.datatype + '._sample_period', FilterChain.datatype + '._sample_rate', FilterChain.operation + '.parameters' ], operations=["==", "like", ">=", "<=", "like"], values=[connectivities[0].gid, "Bold", "500", "0.002", '"conduction_speed": "3.0"' ] ) #### If you want to filter another type of TS, change the kind class bellow, #### instead of TimeSeriesRegion use TimeSeriesEEG, or TimeSeriesSurface, etc. timeseries = _retrieve_entities_by_filters(TimeSeriesRegion, project_id, filter_timeseries) for ts in timeseries: print("=============================") print(ts.summary_info) print(" Original file: " + str(ts.get_storage_file_path())) destination_file = os.path.expanduser(os.path.join(export_folder, ts.get_storage_file_name())) FilesHelper.copy_file(ts.get_storage_file_path(), destination_file) if os.path.exists(destination_file): print(" TS file copied at: " + destination_file) else: print(" Some error happened when trying to copy at destination folder!!")
def _process_input_zip(self, zip_arch, result_folder, remote_prefix, file_name_base, expected_pairs, fix_number=True): """ Read entries in uploaded ZIP. Raise Exception in case pairs HDR/IMG are not matched or number "expected_pairs" is not met. :returns: string with HDR list (to be passed to DTI pipeline). """ hdr_files = [] for file_name in zip_arch.namelist(): if not file_name.startswith(file_name_base) or file_name.endswith("/"): continue if file_name.endswith(".hdr"): pair_img = file_name.replace(".hdr", ".img") if pair_img not in zip_arch.namelist(): raise ConnectException("Could not find pair for HDR file :" + str(file_name)) new_file_name = os.path.join(result_folder, file_name_base + str(len(hdr_files)) + ".hdr") src = zip_arch.open(file_name, 'rU') FilesHelper.copy_file(src, new_file_name) hdr_files.append(os.path.join(remote_prefix, os.path.split(new_file_name)[1])) new_file_name = new_file_name.replace(".hdr", ".img") src = zip_arch.open(pair_img, 'rU') FilesHelper.copy_file(src, new_file_name) elif not file_name.endswith(".img"): self.logger.warning("Ignored file :" + str(file_name)) if len(hdr_files) < expected_pairs or (fix_number and len(hdr_files) > expected_pairs): raise ConnectException("Invalid number of files:" + str(len(hdr_files)) + " expected:" + str(expected_pairs)) result = "" for hdr_name in hdr_files: result = result + hdr_name + " " return result
class ProjectServiceTest(TransactionalTestCase): """ This class contains tests for the tvb.core.services.project_service module. """ def setUp(self): """ Reset the database before each test. """ config.EVENTS_FOLDER = '' self.project_service = ProjectService() self.structure_helper = FilesHelper() self.test_user = TestFactory.create_user() def tearDown(self): """ Remove project folders and clean up database. """ created_projects = dao.get_projects_for_user(self.test_user.id) for project in created_projects: self.structure_helper.remove_project_structure(project.name) self.delete_project_folders() def test_create_project_happy_flow(self): """ Standard flow for creating a new project. """ user1 = TestFactory.create_user('test_user1') user2 = TestFactory.create_user('test_user2') initial_projects = dao.get_projects_for_user(self.test_user.id) self.assertEqual(len(initial_projects), 0, "Database reset probably failed!") TestFactory.create_project(self.test_user, 'test_project', users=[user1.id, user2.id]) resulting_projects = dao.get_projects_for_user(self.test_user.id) self.assertEqual(len(resulting_projects), 1, "Project with valid data not inserted!") project = resulting_projects[0] if project.name == "test_project": self.assertEqual(project.description, "description", "Description do no match") users_for_project = dao.get_members_of_project(project.id) for user in users_for_project: self.assertTrue(user.id in [user1.id, user2.id], "Users not stored properly.") self.assertTrue(os.path.exists(os.path.join(TvbProfile.current.TVB_STORAGE, FilesHelper.PROJECTS_FOLDER, "test_project")), "Folder for project was not created") def test_create_project_empty_name(self): """ Creating a project with an empty name. """ data = dict(name="", description="test_description", users=[]) initial_projects = dao.get_projects_for_user(self.test_user.id) self.assertEqual(len(initial_projects), 0, "Database reset probably failed!") self.assertRaises(ProjectServiceException, self.project_service.store_project, self.test_user, True, None, **data) def test_edit_project_happy_flow(self): """ Standard flow for editing an existing project. """ selected_project = TestFactory.create_project(self.test_user, 'test_proj') proj_root = self.structure_helper.get_project_folder(selected_project) initial_projects = dao.get_projects_for_user(self.test_user.id) self.assertEqual(len(initial_projects), 1, "Database initialization probably failed!") edited_data = dict(name="test_project", description="test_description", users=[]) edited_project = self.project_service.store_project(self.test_user, False, selected_project.id, **edited_data) self.assertFalse(os.path.exists(proj_root), "Previous folder not deleted") proj_root = self.structure_helper.get_project_folder(edited_project) self.assertTrue(os.path.exists(proj_root), "New folder not created!") self.assertNotEqual(selected_project.name, edited_project.name, "Project was no changed!") def test_edit_project_unexisting(self): """ Trying to edit an un-existing project. """ selected_project = TestFactory.create_project(self.test_user, 'test_proj') self.structure_helper.get_project_folder(selected_project) initial_projects = dao.get_projects_for_user(self.test_user.id) self.assertEqual(len(initial_projects), 1, "Database initialization probably failed!") data = dict(name="test_project", description="test_description", users=[]) self.assertRaises(ProjectServiceException, self.project_service.store_project, self.test_user, False, 99, **data) def test_find_project_happy_flow(self): """ Standard flow for finding a project by it's id. """ initial_projects = dao.get_projects_for_user(self.test_user.id) self.assertEqual(len(initial_projects), 0, "Database reset probably failed!") inserted_project = TestFactory.create_project(self.test_user, 'test_project') self.assertTrue(self.project_service.find_project(inserted_project.id) is not None, "Project not found !") dao_returned_project = dao.get_project_by_id(inserted_project.id) service_returned_project = self.project_service.find_project(inserted_project.id) self.assertEqual(dao_returned_project.id, service_returned_project.id, "Data returned from service is different from data returned by DAO.") self.assertEqual(dao_returned_project.name, service_returned_project.name, "Data returned from service is different than data returned by DAO.") self.assertEqual(dao_returned_project.description, service_returned_project.description, "Data returned from service is different from data returned by DAO.") self.assertEqual(dao_returned_project.members, service_returned_project.members, "Data returned from service is different from data returned by DAO.") def test_find_project_unexisting(self): """ Searching for an un-existing project. """ data = dict(name="test_project", description="test_description", users=[]) initial_projects = dao.get_projects_for_user(self.test_user.id) self.assertEqual(len(initial_projects), 0, "Database reset probably failed!") self.project_service.store_project(self.test_user, True, None, **data) self.assertRaises(ProjectServiceException, self.project_service.find_project, 99) def test_retrieve_projects_for_user(self): """ Test for retrieving the projects for a given user. One page only. """ initial_projects = self.project_service.retrieve_projects_for_user(self.test_user.id)[0] self.assertEqual(len(initial_projects), 0, "Database was not reset properly!") TestFactory.create_project(self.test_user, 'test_proj') TestFactory.create_project(self.test_user, 'test_proj1') TestFactory.create_project(self.test_user, 'test_proj2') user1 = TestFactory.create_user('another_user') TestFactory.create_project(user1, 'test_proj3') projects = self.project_service.retrieve_projects_for_user(self.test_user.id)[0] self.assertEqual(len(projects), 3, "Projects not retrieved properly!") for project in projects: self.assertNotEquals(project.name, "test_project3", "This project should not have been retrieved") def test_retrieve_1project_3usr(self): """ One user as admin, two users as members, getting projects for admin and for any of the members should return one. """ member1 = TestFactory.create_user("member1") member2 = TestFactory.create_user("member2") TestFactory.create_project(self.test_user, 'Testproject', users=[member1.id, member2.id]) projects = self.project_service.retrieve_projects_for_user(self.test_user.id, 1)[0] self.assertEqual(len(projects), 1, "Projects not retrieved properly!") projects = self.project_service.retrieve_projects_for_user(member1.id, 1)[0] self.assertEqual(len(projects), 1, "Projects not retrieved properly!") projects = self.project_service.retrieve_projects_for_user(member2.id, 1)[0] self.assertEqual(len(projects), 1, "Projects not retrieved properly!") def test_retrieve_3projects_3usr(self): """ Three users, 3 projects. Structure of db: proj1: {admin: user1, members: [user2, user3]} proj2: {admin: user2, members: [user1]} proj3: {admin: user3, members: [user1, user2]} Check valid project returns for all the users. """ member1 = TestFactory.create_user("member1") member2 = TestFactory.create_user("member2") member3 = TestFactory.create_user("member3") TestFactory.create_project(member1, 'TestProject1', users=[member2.id, member3.id]) TestFactory.create_project(member2, 'TestProject2', users=[member1.id]) TestFactory.create_project(member3, 'TestProject3', users=[member1.id, member2.id]) projects = self.project_service.retrieve_projects_for_user(member1.id, 1)[0] self.assertEqual(len(projects), 3, "Projects not retrieved properly!") projects = self.project_service.retrieve_projects_for_user(member2.id, 1)[0] self.assertEqual(len(projects), 3, "Projects not retrieved properly!") projects = self.project_service.retrieve_projects_for_user(member3.id, 1)[0] self.assertEqual(len(projects), 2, "Projects not retrieved properly!") def test_retrieve_projects_random(self): """ Generate a large number of users/projects, and validate the results. """ ExtremeTestFactory.generate_users(NR_USERS, MAX_PROJ_PER_USER) for i in range(NR_USERS): current_user = dao.get_user_by_name("gen" + str(i)) expected_projects = ExtremeTestFactory.VALIDATION_DICT[current_user.id] if expected_projects % PROJECTS_PAGE_SIZE == 0: expected_pages = expected_projects / PROJECTS_PAGE_SIZE exp_proj_per_page = PROJECTS_PAGE_SIZE else: expected_pages = expected_projects / PROJECTS_PAGE_SIZE + 1 exp_proj_per_page = expected_projects % PROJECTS_PAGE_SIZE if expected_projects == 0: expected_pages = 0 exp_proj_per_page = 0 projects, pages = self.project_service.retrieve_projects_for_user(current_user.id, expected_pages) self.assertEqual(len(projects), exp_proj_per_page, "Projects not retrieved properly! Expected:" + str(exp_proj_per_page) + "but got:" + str(len(projects))) self.assertEqual(pages, expected_pages, "Pages not retrieved properly!") for folder in os.listdir(TvbProfile.current.TVB_STORAGE): full_path = os.path.join(TvbProfile.current.TVB_STORAGE, folder) if os.path.isdir(full_path) and folder.startswith('Generated'): shutil.rmtree(full_path) def test_retrieve_projects_page2(self): """ Test for retrieving the second page projects for a given user. """ for i in range(PROJECTS_PAGE_SIZE + 3): TestFactory.create_project(self.test_user, 'test_proj' + str(i)) projects, pages = self.project_service.retrieve_projects_for_user(self.test_user.id, 2) self.assertEqual(len(projects), (PROJECTS_PAGE_SIZE + 3) % PROJECTS_PAGE_SIZE, "Pagination inproper.") self.assertEqual(pages, 2, 'Wrong number of pages retrieved.') def test_retrieve_projects_and_del(self): """ Test for retrieving the second page projects for a given user. """ created_projects = [] for i in range(PROJECTS_PAGE_SIZE + 1): created_projects.append(TestFactory.create_project(self.test_user, 'test_proj' + str(i))) projects, pages = self.project_service.retrieve_projects_for_user(self.test_user.id, 2) self.assertEqual(len(projects), (PROJECTS_PAGE_SIZE + 1) % PROJECTS_PAGE_SIZE, "Pagination improper.") self.assertEqual(pages, (PROJECTS_PAGE_SIZE + 1) / PROJECTS_PAGE_SIZE + 1, 'Wrong number of pages') self.project_service.remove_project(created_projects[1].id) projects, pages = self.project_service.retrieve_projects_for_user(self.test_user.id, 2) self.assertEqual(len(projects), 0, "Pagination improper.") self.assertEqual(pages, 1, 'Wrong number of pages retrieved.') projects, pages = self.project_service.retrieve_projects_for_user(self.test_user.id, 1) self.assertEqual(len(projects), PROJECTS_PAGE_SIZE, "Pagination improper.") self.assertEqual(pages, 1, 'Wrong number of pages retrieved.') def test_empty_project_has_zero_disk_size(self): TestFactory.create_project(self.test_user, 'test_proj') projects, pages = self.project_service.retrieve_projects_for_user(self.test_user.id) self.assertEqual(0, projects[0].disk_size) self.assertEqual('0.0 KiB', projects[0].disk_size_human) def test_project_disk_size(self): project1 = TestFactory.create_project(self.test_user, 'test_proj1') zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_66.zip') TestFactory.import_zip_connectivity(self.test_user, project1, 'testSubject', zip_path) project2 = TestFactory.create_project(self.test_user, 'test_proj2') TestFactory.import_cff(test_user=self.test_user, test_project=project2) projects = self.project_service.retrieve_projects_for_user(self.test_user.id)[0] self.assertNotEqual(projects[0].disk_size, projects[1].disk_size, "projects should have different size") for project in projects: self.assertNotEqual(0, project.disk_size) self.assertNotEqual('0.0 KiB', project.disk_size_human) prj_folder = self.structure_helper.get_project_folder(project) actual_disk_size = self.compute_recursive_h5_disk_usage(prj_folder)[0] ratio = float(actual_disk_size) / project.disk_size msg = "Real disk usage: %s The one recorded in the db : %s" % (actual_disk_size, project.disk_size) self.assertTrue(ratio < 1.4, msg) def test_get_linkable_projects(self): """ Test for retrieving the projects for a given user. """ initial_projects = self.project_service.retrieve_projects_for_user(self.test_user.id)[0] self.assertEqual(len(initial_projects), 0, "Database was not reset!") test_proj = [] user1 = TestFactory.create_user("another_user") for i in range(4): test_proj.append(TestFactory.create_project(self.test_user if i < 3 else user1, 'test_proj' + str(i))) project_storage = self.structure_helper.get_project_folder(test_proj[0]) operation = TestFactory.create_operation(test_user=self.test_user, test_project=test_proj[0]) project_storage = os.path.join(project_storage, str(operation.id)) os.makedirs(project_storage) datatype = dao.store_entity(model.DataType(module="test_data", subject="subj1", state="test_state", operation_id=operation.id)) linkable = self.project_service.get_linkable_projects_for_user(self.test_user.id, str(datatype.id))[0] self.assertEqual(len(linkable), 2, "Wrong count of link-able projects!") proj_names = [project.name for project in linkable] self.assertTrue(test_proj[1].name in proj_names) self.assertTrue(test_proj[2].name in proj_names) self.assertFalse(test_proj[3].name in proj_names) def test_remove_project_happy_flow(self): """ Standard flow for deleting a project. """ inserted_project = TestFactory.create_project(self.test_user, 'test_proj') project_root = self.structure_helper.get_project_folder(inserted_project) projects = dao.get_projects_for_user(self.test_user.id) self.assertEqual(len(projects), 1, "Initializations failed!") self.assertTrue(os.path.exists(project_root), "Something failed at insert time!") self.project_service.remove_project(inserted_project.id) projects = dao.get_projects_for_user(self.test_user.id) self.assertEqual(len(projects), 0, "Project was not deleted!") self.assertFalse(os.path.exists(project_root), "Root folder not deleted!") def test_remove_project_wrong_id(self): """ Flow for deleting a project giving an un-existing id. """ TestFactory.create_project(self.test_user, 'test_proj') projects = dao.get_projects_for_user(self.test_user.id) self.assertEqual(len(projects), 1, "Initializations failed!") self.assertRaises(ProjectServiceException, self.project_service.remove_project, 99) @staticmethod def _create_value_wrapper(test_user, test_project=None): """ Creates a ValueWrapper dataType, and the associated parent Operation. This is also used in ProjectStructureTest. """ if test_project is None: test_project = TestFactory.create_project(test_user, 'test_proj') operation = TestFactory.create_operation(test_user=test_user, test_project=test_project) value_wrapper = ValueWrapper(data_value=5.0, data_name="my_value") value_wrapper.type = "ValueWrapper" value_wrapper.module = "tvb.datatypes.mapped_values" value_wrapper.subject = "John Doe" value_wrapper.state = "RAW_STATE" value_wrapper.set_operation_id(operation.id) adapter_instance = StoreAdapter([value_wrapper]) OperationService().initiate_prelaunch(operation, adapter_instance, {}) all_value_wrappers = FlowService().get_available_datatypes(test_project.id, "tvb.datatypes.mapped_values.ValueWrapper")[0] if len(all_value_wrappers) != 1: raise Exception("Should be only one value wrapper.") result_vw = ABCAdapter.load_entity_by_gid(all_value_wrappers[0][2]) return test_project, result_vw.gid, operation.gid def __check_meta_data(self, expected_meta_data, new_datatype): """Validate Meta-Data""" mapp_keys = {DataTypeMetaData.KEY_SUBJECT: "subject", DataTypeMetaData.KEY_STATE: "state"} for key, value in expected_meta_data.iteritems(): if key in mapp_keys: self.assertEqual(value, getattr(new_datatype, mapp_keys[key])) elif key == DataTypeMetaData.KEY_OPERATION_TAG: if DataTypeMetaData.KEY_OP_GROUP_ID in expected_meta_data: ## We have a Group to check op_group = new_datatype.parent_operation.fk_operation_group op_group = dao.get_generic_entity(model.OperationGroup, op_group)[0] self.assertEqual(value, op_group.name) else: self.assertEqual(value, new_datatype.parent_operation.user_group) def test_remove_project_node(self): """ Test removing of a node from a project. """ inserted_project, gid, gid_op = self._create_value_wrapper(self.test_user) project_to_link = model.Project("Link", self.test_user.id, "descript") project_to_link = dao.store_entity(project_to_link) exact_data = dao.get_datatype_by_gid(gid) dao.store_entity(model.Links(exact_data.id, project_to_link.id)) self.assertTrue(dao.get_datatype_by_gid(gid) is not None, "Initialization problem!") operation_id = dao.get_generic_entity(model.Operation, gid_op, 'gid')[0].id op_folder = self.structure_helper.get_project_folder("test_proj", str(operation_id)) self.assertTrue(os.path.exists(op_folder)) sub_files = os.listdir(op_folder) self.assertEqual(2, len(sub_files)) ### Validate that no more files are created than needed. self.project_service._remove_project_node_files(inserted_project.id, gid) sub_files = os.listdir(op_folder) self.assertEqual(1, len(sub_files)) ### operation.xml file should still be there op_folder = self.structure_helper.get_project_folder("Link", str(operation_id + 1)) sub_files = os.listdir(op_folder) self.assertEqual(2, len(sub_files)) self.assertTrue(dao.get_datatype_by_gid(gid) is not None, "Data should still be in DB, because of links") self.project_service._remove_project_node_files(project_to_link.id, gid) self.assertTrue(dao.get_datatype_by_gid(gid) is None) sub_files = os.listdir(op_folder) self.assertEqual(1, len(sub_files)) ### operation.xml file should still be there def test_update_meta_data_simple(self): """ Test the new update metaData for a simple data that is not part of a group. """ inserted_project, gid, _ = self._create_value_wrapper(self.test_user) new_meta_data = {DataTypeOverlayDetails.DATA_SUBJECT: "new subject", DataTypeOverlayDetails.DATA_STATE: "second_state", DataTypeOverlayDetails.CODE_GID: gid, DataTypeOverlayDetails.CODE_OPERATION_TAG: 'new user group'} self.project_service.update_metadata(new_meta_data) new_datatype = dao.get_datatype_by_gid(gid) self.__check_meta_data(new_meta_data, new_datatype) op_path = FilesHelper().get_operation_meta_file_path(inserted_project.name, new_datatype.parent_operation.id) op_meta = XMLReader(op_path).read_metadata() self.assertEqual(op_meta['user_group'], 'new user group', 'UserGroup not updated!') def test_update_meta_data_group(self): """ Test the new update metaData for a group of dataTypes. """ datatypes, group_id = TestFactory.create_group(self.test_user, subject="test-subject-1") new_meta_data = {DataTypeOverlayDetails.DATA_SUBJECT: "new subject", DataTypeOverlayDetails.DATA_STATE: "updated_state", DataTypeOverlayDetails.CODE_OPERATION_GROUP_ID: group_id, DataTypeOverlayDetails.CODE_OPERATION_TAG: 'newGroupName'} self.project_service.update_metadata(new_meta_data) for datatype in datatypes: new_datatype = dao.get_datatype_by_id(datatype.id) self.assertEqual(group_id, new_datatype.parent_operation.fk_operation_group) new_group = dao.get_generic_entity(model.OperationGroup, group_id)[0] self.assertEqual(new_group.name, "newGroupName") self.__check_meta_data(new_meta_data, new_datatype) def _create_datatypes(self, dt_factory, nr_of_dts): for idx in range(nr_of_dts): dt = Datatype1() dt.row1 = "value%i" % (idx,) dt.row2 = "value%i" % (idx + 1,) dt_factory._store_datatype(dt) def test_retrieve_project_full(self): """ Tests full project information is retrieved by method `ProjectService.retrieve_project_full(...)` """ dt_factory = datatypes_factory.DatatypesFactory() self._create_datatypes(dt_factory, 3) _, ops_nr, operations, pages_no = self.project_service.retrieve_project_full(dt_factory.project.id) self.assertEqual(ops_nr, 1, "DataType Factory should only use one operation to store all it's datatypes.") self.assertEqual(pages_no, 1, "DataType Factory should only use one operation to store all it's datatypes.") resulted_dts = operations[0]['results'] self.assertEqual(len(resulted_dts), 3, "3 datatypes should be created.") def test_get_project_structure(self): """ Tests project structure is as expected and contains all datatypes """ SELF_DTS_NUMBER = 3 dt_factory_1 = datatypes_factory.DatatypesFactory() self._create_datatypes(dt_factory_1, SELF_DTS_NUMBER) dt_group = dt_factory_1.create_datatype_group() link_ids, expected_links = [], [] # Prepare link towards a simple DT dt_factory_2 = datatypes_factory.DatatypesFactory() dt_to_link = dt_factory_2.create_simple_datatype() link_ids.append(dt_to_link.id) expected_links.append(dt_to_link.gid) # Prepare links towards a full DT Group, but expecting only the DT_Group in the final tree link_gr = dt_factory_2.create_datatype_group() dts = dao.get_datatype_in_group(datatype_group_id=link_gr.id) link_ids.extend([dt_to_link.id for dt_to_link in dts]) link_ids.append(link_gr.id) expected_links.append(link_gr.gid) # Prepare link towards a single DT inside a group, and expecting to find the DT in the final tree link_gr = dt_factory_2.create_datatype_group() dt_to_link = dao.get_datatype_in_group(datatype_group_id=link_gr.id)[0] link_ids.append(dt_to_link.id) expected_links.append(dt_to_link.gid) # Actually create the links from Prj2 into Prj1 FlowService().create_link(link_ids, dt_factory_1.project.id) # Retrieve the raw data used to compose the tree (for easy parsing) dts_in_tree = dao.get_data_in_project(dt_factory_1.project.id) dts_in_tree = [dt.gid for dt in dts_in_tree] # Retrieve the tree json (for trivial validations only, as we can not decode) node_json = self.project_service.get_project_structure(dt_factory_1.project, None, DataTypeMetaData.KEY_STATE, DataTypeMetaData.KEY_SUBJECT, None) self.assertEqual(len(expected_links) + SELF_DTS_NUMBER + 2, len(dts_in_tree), "invalid number of nodes in tree") self.assertFalse(link_gr.gid in dts_in_tree, "DT_group where a single DT is linked is not expected.") self.assertTrue(dt_group.gid in dts_in_tree, "DT_Group should be in the Project Tree!") self.assertTrue(dt_group.gid in node_json, "DT_Group should be in the Project Tree JSON!") project_dts = dao.get_datatypes_in_project(dt_factory_1.project.id) for dt in project_dts: if dt.fk_datatype_group is not None: self.assertFalse(dt.gid in node_json, "DTs part of a group should not be") self.assertFalse(dt.gid in dts_in_tree, "DTs part of a group should not be") else: self.assertTrue(dt.gid in node_json, "Simple DTs and DT_Groups should be") self.assertTrue(dt.gid in dts_in_tree, "Simple DTs and DT_Groups should be") for link_gid in expected_links: self.assertTrue(link_gid in node_json, "Expected Link not present") self.assertTrue(link_gid in dts_in_tree, "Expected Link not present")
def transactional_setup_method(self): self.figure_service = FigureService() self.user = TestFactory.create_user() self.project = TestFactory.create_project(admin=self.user) self.files_helper = FilesHelper()
class AlgorithmService(object): """ Service Layer for Algorithms manipulation (e.g. find all Uploaders, Filter algo by category, etc) """ def __init__(self): self.logger = get_logger(self.__class__.__module__) self.file_helper = FilesHelper() @staticmethod def get_category_by_id(identifier): """ Pass to DAO the retrieve of category by ID operation.""" return dao.get_category_by_id(identifier) @staticmethod def get_raw_categories(): """:returns: AlgorithmCategory list of entities that have results in RAW state (Creators/Uploaders)""" return dao.get_raw_categories() @staticmethod def get_visualisers_category(): """Retrieve all Algorithm categories, with display capability""" result = dao.get_visualisers_categories() if not result: raise ValueError("View Category not found!!!") return result[0] @staticmethod def get_algorithm_by_identifier(ident): """ Retrieve Algorithm entity by ID. Return None, if ID is not found in DB. """ return dao.get_algorithm_by_id(ident) @staticmethod def get_operation_numbers(proj_id): """ Count total number of operations started for current project. """ return dao.get_operation_numbers(proj_id) def _prepare_dt_display_name(self, dt_index, dt): # dt is a result of the get_values_of_datatype function db_dt = dao.get_generic_entity(dt_index, dt[2], "gid") display_name = db_dt[0].display_name display_name += ' - ' + (dt[3] or "None ") # Subject if dt[5]: display_name += ' - From: ' + str(dt[5]) else: display_name += utils.date2string(dt[4]) if dt[6]: display_name += ' - ' + str(dt[6]) display_name += ' - ID:' + str(dt[0]) return display_name def fill_selectfield_with_datatypes(self, field, project_id, extra_conditions=None): # type: (TraitDataTypeSelectField, int, list) -> None filtering_conditions = FilterChain() filtering_conditions += field.conditions filtering_conditions += extra_conditions datatypes, _ = dao.get_values_of_datatype(project_id, field.datatype_index, filtering_conditions) datatype_options = [] for datatype in datatypes: display_name = self._prepare_dt_display_name( field.datatype_index, datatype) datatype_options.append((datatype, display_name)) field.datatype_options = datatype_options def _fill_form_with_datatypes(self, form, project_id, extra_conditions=None): for form_field in form.trait_fields: if isinstance(form_field, TraitDataTypeSelectField): self.fill_selectfield_with_datatypes(form_field, project_id, extra_conditions) return form def prepare_adapter_form(self, adapter_instance=None, form_instance=None, project_id=None, extra_conditions=None): # type: (ABCAdapter, ABCAdapterForm, int, []) -> ABCAdapterForm form = None if form_instance is not None: form = form_instance elif adapter_instance is not None: form = adapter_instance.get_form()() if form is None: raise OperationException("Cannot prepare None form") form = self._fill_form_with_datatypes(form, project_id, extra_conditions) return form def _prepare_upload_post_data(self, form, post_data, project_id): for form_field in form.trait_fields: if isinstance(form_field, TraitUploadField) and form_field.name in post_data: field = post_data[form_field.name] file_name = None if hasattr(field, 'file') and field.file is not None: project = dao.get_project_by_id(project_id) temporary_storage = self.file_helper.get_project_folder( project, self.file_helper.TEMP_FOLDER) try: uq_name = utils.date2string(datetime.now(), True) + '_' + str(0) file_name = TEMPORARY_PREFIX + uq_name + '_' + field.filename file_name = os.path.join(temporary_storage, file_name) with open(file_name, 'wb') as file_obj: file_obj.write(field.file.read()) except Exception as excep: # TODO: is this handled properly? self.file_helper.remove_files([file_name]) excep.message = 'Could not continue: Invalid input files' raise excep post_data[form_field.name] = file_name def fill_adapter_form(self, adapter_instance, post_data, project_id): # type: (ABCAdapter, dict, int) -> ABCAdapterForm form = self.prepare_adapter_form(adapter_instance=adapter_instance, project_id=project_id) if isinstance(form, ABCUploaderForm): self._prepare_upload_post_data(form, post_data, project_id) if 'fill_defaults' in post_data: form.fill_from_post_plus_defaults(post_data) else: form.fill_from_post(post_data) return form def prepare_adapter(self, stored_adapter): adapter_module = stored_adapter.module adapter_name = stored_adapter.classname try: # Prepare Adapter Interface, by populating with existent data, # in case of a parameter of type DataType. adapter_instance = ABCAdapter.build_adapter(stored_adapter) return adapter_instance except Exception: self.logger.exception('Not found:' + adapter_name + ' in:' + adapter_module) raise OperationException("Could not prepare " + adapter_name) @staticmethod def get_algorithm_by_module_and_class(module, classname): """ Get the db entry from the algorithm table for the given module and class. """ return dao.get_algorithm_by_module(module, classname) @staticmethod def create_link(data_ids, project_id): """ For a list of dataType IDs and a project id create all the required links. """ for data in data_ids: link = Links(data, project_id) dao.store_entity(link) @staticmethod def remove_link(dt_id, project_id): """ Remove the link from the datatype given by dt_id to project given by project_id. """ link = dao.get_link(dt_id, project_id) if link is not None: dao.remove_entity(Links, link.id) @staticmethod def get_upload_algorithms(): """ :return: List of StoredAdapter entities """ categories = dao.get_uploader_categories() categories_ids = [categ.id for categ in categories] return dao.get_adapters_from_categories(categories_ids) @staticmethod def get_analyze_groups(): """ :return: list of AlgorithmTransientGroup entities """ categories = dao.get_launchable_categories(elimin_viewers=True) categories_ids = [categ.id for categ in categories] stored_adapters = dao.get_adapters_from_categories(categories_ids) groups_list = [] for adapter in stored_adapters: # For empty groups, this time, we fill the actual adapter group = AlgorithmTransientGroup( adapter.group_name or adapter.displayname, adapter.group_description or adapter.description) group = AlgorithmService._find_group(groups_list, group) group.children.append(adapter) return categories[0], groups_list @staticmethod def _find_group(groups_list, new_group): for i in range(len(groups_list) - 1, -1, -1): current_group = groups_list[i] if current_group.name == new_group.name and current_group.description == new_group.description: return current_group # Not found in list groups_list.append(new_group) return new_group def get_visualizers_for_group(self, dt_group_gid): categories = dao.get_visualisers_categories() return self._get_launchable_algorithms(dt_group_gid, categories)[1] def get_launchable_algorithms(self, datatype_gid): """ :param datatype_gid: Filter only algorithms compatible with this GUID :return: dict(category_name: List AlgorithmTransientGroup) """ categories = dao.get_launchable_categories() datatype_instance, filtered_adapters, has_operations_warning = self._get_launchable_algorithms( datatype_gid, categories) categories_dict = dict() for c in categories: categories_dict[c.id] = c.displayname return self._group_adapters_by_category( filtered_adapters, categories_dict), has_operations_warning def _get_launchable_algorithms(self, datatype_gid, categories): datatype_instance = dao.get_datatype_by_gid(datatype_gid) return self.get_launchable_algorithms_for_datatype( datatype_instance, categories) def get_launchable_algorithms_for_datatype(self, datatype, categories): data_class = datatype.__class__ all_compatible_classes = [data_class.__name__] for one_class in getmro(data_class): # from tvb.basic.traits.types_mapped import MappedType if issubclass( one_class, DataType ) and one_class.__name__ not in all_compatible_classes: all_compatible_classes.append(one_class.__name__) self.logger.debug("Searching in categories: " + str(categories) + " for classes " + str(all_compatible_classes)) categories_ids = [categ.id for categ in categories] launchable_adapters = dao.get_applicable_adapters( all_compatible_classes, categories_ids) filtered_adapters = [] has_operations_warning = False for stored_adapter in launchable_adapters: filter_chain = FilterChain.from_json( stored_adapter.datatype_filter) try: if not filter_chain or filter_chain.get_python_filter_equivalent( datatype): filtered_adapters.append(stored_adapter) except (TypeError, InvalidFilterChainInput): self.logger.exception("Could not evaluate filter on " + str(stored_adapter)) has_operations_warning = True return datatype, filtered_adapters, has_operations_warning def _group_adapters_by_category(self, stored_adapters, categories): """ :param stored_adapters: list StoredAdapter :return: dict(category_name: List AlgorithmTransientGroup), empty groups all in the same AlgorithmTransientGroup """ categories_dict = dict() for adapter in stored_adapters: category_name = categories.get(adapter.fk_category) if category_name in categories_dict: groups_list = categories_dict.get(category_name) else: groups_list = [] categories_dict[category_name] = groups_list group = AlgorithmTransientGroup(adapter.group_name, adapter.group_description) group = self._find_group(groups_list, group) group.children.append(adapter) return categories_dict @staticmethod def get_generic_entity(entity_type, filter_value, select_field): return dao.get_generic_entity(entity_type, filter_value, select_field) ########################################################################## ######## Methods below are for MeasurePoint selections ################### ########################################################################## @staticmethod def get_selections_for_project(project_id, datatype_gid): """ Retrieved from DB saved selections for current project. If a certain selection doesn't have all the labels between the labels of the given connectivity than this selection will not be returned. :returns: List of ConnectivitySelection entities. """ return dao.get_selections_for_project(project_id, datatype_gid) @staticmethod def save_measure_points_selection(ui_name, selected_nodes, datatype_gid, project_id): """ Store in DB a ConnectivitySelection. """ select_entities = dao.get_selections_for_project( project_id, datatype_gid, ui_name) if select_entities: # when the name of the new selection is within the available selections then update that selection: select_entity = select_entities[0] select_entity.selected_nodes = selected_nodes else: select_entity = MeasurePointsSelection(ui_name, selected_nodes, datatype_gid, project_id) dao.store_entity(select_entity) ########################################################################## ########## Bellow are PSE Filters specific methods ################## ########################################################################## @staticmethod def get_stored_pse_filters(datatype_group_gid): return dao.get_stored_pse_filters(datatype_group_gid) @staticmethod def save_pse_filter(ui_name, datatype_group_gid, threshold_value, applied_on): """ Store in DB a PSE filter. """ select_entities = dao.get_stored_pse_filters(datatype_group_gid, ui_name) if select_entities: # when the UI name is already in DB, update the existing entity select_entity = select_entities[0] select_entity.threshold_value = threshold_value select_entity.applied_on = applied_on # this is the type, as in applied on size or color else: select_entity = StoredPSEFilter(ui_name, datatype_group_gid, threshold_value, applied_on) dao.store_entity(select_entity)
def test_get_filtered_datatypes(self): """ Test the filter function when retrieving dataTypes. """ #Create some test operations start_dates = [ datetime.now(), datetime.strptime("08-06-2010", "%m-%d-%Y"), datetime.strptime("07-21-2010", "%m-%d-%Y"), datetime.strptime("05-06-2010", "%m-%d-%Y"), datetime.strptime("07-21-2011", "%m-%d-%Y") ] end_dates = [ datetime.now(), datetime.strptime("08-12-2010", "%m-%d-%Y"), datetime.strptime("08-12-2010", "%m-%d-%Y"), datetime.strptime("08-12-2011", "%m-%d-%Y"), datetime.strptime("08-12-2011", "%m-%d-%Y") ] for i in range(5): operation = model.Operation(self.test_user.id, self.test_project.id, self.algorithm.id, 'test params', status=model.STATUS_FINISHED, start_date=start_dates[i], completion_date=end_dates[i]) operation = dao.store_entity(operation) storage_path = FilesHelper().get_project_folder( self.test_project, str(operation.id)) if i < 4: datatype_inst = Datatype1() datatype_inst.type = "Datatype1" datatype_inst.subject = "John Doe" + str(i) datatype_inst.state = "RAW" datatype_inst.set_operation_id(operation.id) dao.store_entity(datatype_inst) else: for _ in range(2): datatype_inst = Datatype2() datatype_inst.storage_path = storage_path datatype_inst.type = "Datatype2" datatype_inst.subject = "John Doe" + str(i) datatype_inst.state = "RAW" datatype_inst.string_data = ["data"] datatype_inst.set_operation_id(operation.id) dao.store_entity(datatype_inst) returned_data = self.flow_service.get_available_datatypes( self.test_project.id, Datatype1)[0] for row in returned_data: if row[1] != 'Datatype1': self.fail("Some invalid data was returned!") self.assertEqual(4, len(returned_data), "Invalid length of result") filter_op = FilterChain( fields=[ FilterChain.datatype + ".state", FilterChain.operation + ".start_date" ], values=["RAW", datetime.strptime("08-01-2010", "%m-%d-%Y")], operations=["==", ">"]) returned_data = self.flow_service.get_available_datatypes( self.test_project.id, Datatype1, filter_op)[0] returned_subjects = [one_data[3] for one_data in returned_data] if "John Doe0" not in returned_subjects or "John Doe1" not in returned_subjects or len( returned_subjects) != 2: self.fail("DataTypes were not filtered properly!")
def __init__(self): BaseController.__init__(self) self.context = SelectedAdapterContext() self.files_helper = FilesHelper()
def __init__(self): self.logger = get_logger(__name__) self.structure_helper = FilesHelper()
class TestFigureService(TransactionalTestCase): """ Tests for the figure service """ def transactional_setup_method(self): self.figure_service = FigureService() self.user = TestFactory.create_user() self.project = TestFactory.create_project(admin=self.user) self.files_helper = FilesHelper() def transactional_teardown_method(self): self.delete_project_folders() def assertCanReadImage(self, image_path): try: Image.open(image_path).load() except (IOError, ValueError): raise AssertionError("Could not open %s as a image" % image_path) def store_test_png(self): self.figure_service.store_result_figure(self.project, self.user, "png", IMG_DATA, image_name="test-figure") def retrieve_images(self): figures_by_session, _ = self.figure_service.retrieve_result_figures( self.project, self.user) # flatten image session grouping figures = [] for fg in figures_by_session.itervalues(): figures.extend(fg) return figures def test_store_image(self): self.store_test_png() def test_store_image_from_operation(self): # test that image can be retrieved from operation test_operation = TestFactory.create_operation( test_user=self.user, test_project=self.project) self.figure_service.store_result_figure(self.project, self.user, "png", IMG_DATA, operation_id=test_operation.id) figures = dao.get_figures_for_operation(test_operation.id) assert 1 == len(figures) image_path = self.files_helper.get_images_folder(self.project.name) image_path = os.path.join(image_path, figures[0].file_path) self.assertCanReadImage(image_path) def test_store_and_retrieve_image(self): self.store_test_png() figures = self.retrieve_images() assert 1 == len(figures) image_path = utils.url2path(figures[0].file_path) self.assertCanReadImage(image_path) def test_load_figure(self): self.store_test_png() figures = self.retrieve_images() self.figure_service.load_figure(figures[0].id) def test_edit_figure(self): session_name = 'the altered ones' name = 'altered' self.store_test_png() figures = self.retrieve_images() self.figure_service.edit_result_figure(figures[0].id, session_name=session_name, name=name) figures_by_session, _ = self.figure_service.retrieve_result_figures( self.project, self.user) assert [session_name] == figures_by_session.keys() assert name == figures_by_session.values()[0][0].name def test_remove_figure(self): self.store_test_png() figures = self.retrieve_images() assert 1 == len(figures) self.figure_service.remove_result_figure(figures[0].id) figures = self.retrieve_images() assert 0 == len(figures)
def transactional_teardown_method(self): FilesHelper().remove_project_structure(self.test_project.name)
def transactional_teardown_method(self): """ Clean-up tests data """ FilesHelper().remove_project_structure(self.test_project.name)
def export_project(self, project, optimize_size=False): """ Given a project root and the TVB storage_path, create a ZIP ready for export. :param project: project object which identifies project to be exported """ if project is None: raise ExportException("Please provide project to be exported") files_helper = FilesHelper() project_folder = files_helper.get_project_folder(project) project_datatypes = dao.get_datatypes_in_project( project.id, only_visible=optimize_size) to_be_exported_folders = [] considered_op_ids = [] if optimize_size: # take only the DataType with visibility flag set ON for dt in project_datatypes: op_id = dt.fk_from_operation if op_id not in considered_op_ids: to_be_exported_folders.append({ 'folder': files_helper.get_project_folder(project, str(op_id)), 'archive_path_prefix': str(op_id) + os.sep }) considered_op_ids.append(op_id) else: to_be_exported_folders.append({ 'folder': project_folder, 'archive_path_prefix': '', 'exclude': ["TEMP"] }) # Compute path and name of the zip file now = datetime.now() date_str = now.strftime("%Y-%m-%d_%H-%M") zip_file_name = "%s_%s.%s" % (date_str, project.name, self.ZIP_FILE_EXTENSION) export_folder = self._build_data_export_folder(project) result_path = os.path.join(export_folder, zip_file_name) with TvbZip(result_path, "w") as zip_file: # Pack project [filtered] content into a ZIP file: self.logger.debug("Done preparing, now we will write folders " + str(len(to_be_exported_folders))) self.logger.debug(str(to_be_exported_folders)) for pack in to_be_exported_folders: zip_file.write_folder(**pack) self.logger.debug( "Done exporting files, now we will export linked DTs") self._export_linked_datatypes(project, zip_file) # Make sure the Project.xml file gets copied: if optimize_size: self.logger.debug("Done linked, now we write the project xml") zip_file.write( files_helper.get_project_meta_file_path(project.name), files_helper.TVB_PROJECT_FILE) self.logger.debug("Done, closing") return result_path
def __init__(self): self.logger = get_logger(self.__class__.__module__) self.burst_service = BurstService() self.operation_service = OperationService() self.files_helper = FilesHelper()
class SimulatorService(object): def __init__(self): self.logger = get_logger(self.__class__.__module__) self.burst_service = BurstService() self.operation_service = OperationService() self.files_helper = FilesHelper() def _reset_model(self, session_stored_simulator): session_stored_simulator.model = type(session_stored_simulator.model)() vi_indexes = MonitorForm.determine_indexes_for_chosen_vars_of_interest( session_stored_simulator) vi_indexes = numpy.array(list(vi_indexes.values())) for monitor in session_stored_simulator.monitors: monitor.variables_of_interest = vi_indexes def reset_at_connectivity_change(self, is_simulator_copy, form, session_stored_simulator): """ In case the user copies a simulation and changes the Connectivity, we want to reset the Model and Noise parameters because they might not fit to the new Connectivity's nr of regions. """ if is_simulator_copy and form.connectivity.value != session_stored_simulator.connectivity: self._reset_model(session_stored_simulator) if issubclass(type(session_stored_simulator.integrator), IntegratorStochastic): session_stored_simulator.integrator.noise = type( session_stored_simulator.integrator.noise)() def reset_at_surface_change(self, is_simulator_copy, form, session_stored_simulator): """ In case the user copies a surface-simulation and changes the Surface, we want to reset the Model parameters because they might not fit to the new Surface's nr of vertices. """ if is_simulator_copy and ( session_stored_simulator.surface is None and form.surface.value or session_stored_simulator.surface and form.surface.value != session_stored_simulator.surface.surface_gid): self._reset_model(session_stored_simulator) @staticmethod def _set_simulator_range_parameter(simulator, range_parameter_name, range_parameter_value): range_param_name_list = range_parameter_name.split('.') current_attr = simulator for param_name in range_param_name_list[:len(range_param_name_list) - 1]: current_attr = getattr(current_attr, param_name) setattr(current_attr, range_param_name_list[-1], range_parameter_value) def async_launch_and_prepare_simulation(self, burst_config, user, project, simulator_algo, session_stored_simulator): try: operation = self.operation_service.prepare_operation( user.id, project.id, simulator_algo, session_stored_simulator.gid) ga = self.operation_service._prepare_metadata( simulator_algo.algorithm_category, {}, None, burst_config.gid) session_stored_simulator.generic_attributes = ga storage_path = self.files_helper.get_project_folder( project, str(operation.id)) h5.store_view_model(session_stored_simulator, storage_path) burst_config = self.burst_service.update_simulation_fields( burst_config.id, operation.id, session_stored_simulator.gid) self.burst_service.store_burst_configuration( burst_config, storage_path) wf_errs = 0 try: OperationService().launch_operation(operation.id, True) return operation except Exception as excep: self.logger.error(excep) wf_errs += 1 if burst_config: self.burst_service.mark_burst_finished( burst_config, error_message=str(excep)) self.logger.debug( "Finished launching workflow. The operation was launched successfully, " + str(wf_errs) + " had error on pre-launch steps") except Exception as excep: self.logger.error(excep) if burst_config: self.burst_service.mark_burst_finished( burst_config, error_message=str(excep)) def prepare_simulation_on_server(self, user_id, project, algorithm, zip_folder_path, simulator_file): simulator_vm = h5.load_view_model_from_file(simulator_file) operation = self.operation_service.prepare_operation( user_id, project.id, algorithm, simulator_vm.gid) storage_operation_path = self.files_helper.get_project_folder( project, str(operation.id)) self.async_launch_simulation_on_server(operation, zip_folder_path, storage_operation_path) return operation def async_launch_simulation_on_server(self, operation, zip_folder_path, storage_operation_path): try: for file in os.listdir(zip_folder_path): shutil.move(os.path.join(zip_folder_path, file), storage_operation_path) try: OperationService().launch_operation(operation.id, True) shutil.rmtree(zip_folder_path) return operation except Exception as excep: self.logger.error(excep) except Exception as excep: self.logger.error(excep) @staticmethod def _set_range_param_in_dict(param_value): if type(param_value) is numpy.ndarray: return param_value[0] elif isinstance(param_value, uuid.UUID): return param_value.hex else: return param_value def async_launch_and_prepare_pse(self, burst_config, user, project, simulator_algo, range_param1, range_param2, session_stored_simulator): try: algo_category = simulator_algo.algorithm_category operation_group = burst_config.operation_group metric_operation_group = burst_config.metric_operation_group operations = [] range_param2_values = [None] if range_param2: range_param2_values = range_param2.get_range_values() first_simulator = None for param1_value in range_param1.get_range_values(): for param2_value in range_param2_values: # Copy, but generate a new GUID for every Simulator in PSE simulator = copy.deepcopy(session_stored_simulator) simulator.gid = uuid.uuid4() self._set_simulator_range_parameter( simulator, range_param1.name, param1_value) ranges = { range_param1.name: self._set_range_param_in_dict(param1_value) } if param2_value is not None: self._set_simulator_range_parameter( simulator, range_param2.name, param2_value) ranges[ range_param2.name] = self._set_range_param_in_dict( param2_value) ranges = json.dumps(ranges) operation = self.operation_service.prepare_operation( user.id, project.id, simulator_algo, simulator.gid, operation_group, ranges) storage_path = self.files_helper.get_project_folder( project, str(operation.id)) h5.store_view_model(simulator, storage_path) operations.append(operation) if first_simulator is None: first_simulator = simulator first_operation = operations[0] storage_path = self.files_helper.get_project_folder( project, str(first_operation.id)) burst_config = self.burst_service.update_simulation_fields( burst_config.id, first_operation.id, first_simulator.gid) self.burst_service.store_burst_configuration( burst_config, storage_path) datatype_group = DataTypeGroup( operation_group, operation_id=first_operation.id, fk_parent_burst=burst_config.gid, state=algo_category.defaultdatastate) dao.store_entity(datatype_group) metrics_datatype_group = DataTypeGroup( metric_operation_group, fk_parent_burst=burst_config.gid) dao.store_entity(metrics_datatype_group) wf_errs = 0 for operation in operations: try: OperationService().launch_operation(operation.id, True) except Exception as excep: self.logger.error(excep) wf_errs += 1 self.burst_service.mark_burst_finished( burst_config, error_message=str(excep)) self.logger.debug("Finished launching workflows. " + str(len(operations) - wf_errs) + " were launched successfully, " + str(wf_errs) + " had error on pre-launch steps") return first_operation except Exception as excep: self.logger.error(excep) self.burst_service.mark_burst_finished(burst_config, error_message=str(excep)) def load_from_zip(self, zip_file, project): import_service = ImportService() simulator_folder = import_service.import_simulator_configuration_zip( zip_file) simulator_h5_filename = DirLoader( simulator_folder, None).find_file_for_has_traits_type(SimulatorAdapterModel) simulator_h5_filepath = os.path.join(simulator_folder, simulator_h5_filename) simulator = h5.load_view_model_from_file(simulator_h5_filepath) burst_config = self.burst_service.load_burst_configuration_from_folder( simulator_folder, project) return simulator, burst_config
class FlowService: """ Service Layer for all TVB generic Work-Flow operations. """ def __init__(self): self.logger = get_logger(self.__class__.__module__) self.file_helper = FilesHelper() self.input_tree_manager = InputTreeManager() def get_category_by_id(self, identifier): """ Pass to DAO the retrieve of category by ID operation.""" return dao.get_category_by_id(identifier) @staticmethod def get_raw_categories(): """:returns: AlgorithmCategory list of entities that have results in RAW state (Creators/Uploaders)""" return dao.get_raw_categories() @staticmethod def get_visualisers_category(): """Retrieve all Algorithm categories, with display capability""" result = dao.get_visualisers_categories() if not result: raise ValueError("View Category not found!!!") return result[0] @staticmethod def get_algorithm_by_identifier(ident): """ Retrieve Algorithm entity by ID. Return None, if ID is not found in DB. """ return dao.get_algorithm_by_id(ident) @staticmethod def load_operation(operation_id): """ Retrieve previously stored Operation from DB, and load operation.burst attribute""" operation = dao.get_operation_by_id(operation_id) operation.burst = dao.get_burst_for_operation_id(operation_id) return operation @staticmethod def get_operation_numbers(proj_id): """ Count total number of operations started for current project. """ return dao.get_operation_numbers(proj_id) def prepare_adapter(self, project_id, stored_adapter): """ Having a StoredAdapter, return the Tree Adapter Interface object, populated with datatypes from 'project_id'. """ adapter_module = stored_adapter.module adapter_name = stored_adapter.classname try: # Prepare Adapter Interface, by populating with existent data, # in case of a parameter of type DataType. adapter_instance = ABCAdapter.build_adapter(stored_adapter) interface = adapter_instance.get_input_tree() interface = self.input_tree_manager.fill_input_tree_with_options(interface, project_id, stored_adapter.fk_category) interface = self.input_tree_manager.prepare_param_names(interface) return interface except Exception: self.logger.exception('Not found:' + adapter_name + ' in:' + adapter_module) raise OperationException("Could not prepare " + adapter_name) @staticmethod def get_algorithm_by_module_and_class(module, classname): """ Get the db entry from the algorithm table for the given module and class. """ return dao.get_algorithm_by_module(module, classname) @staticmethod def get_available_datatypes(project_id, data_type_cls, filters=None): """ Return all dataTypes that match a given name and some filters. :param data_type_cls: either a fully qualified class name or a class object """ return get_filtered_datatypes(project_id, data_type_cls, filters) @staticmethod def create_link(data_ids, project_id): """ For a list of dataType IDs and a project id create all the required links. """ for data in data_ids: link = model.Links(data, project_id) dao.store_entity(link) @staticmethod def remove_link(dt_id, project_id): """ Remove the link from the datatype given by dt_id to project given by project_id. """ link = dao.get_link(dt_id, project_id) if link is not None: dao.remove_entity(model.Links, link.id) def fire_operation(self, adapter_instance, current_user, project_id, visible=True, **data): """ Launch an operation, specified by AdapterInstance, for CurrentUser, Current Project and a given set of UI Input Data. """ operation_name = str(adapter_instance.__class__.__name__) try: self.logger.info("Starting operation " + operation_name) project = dao.get_project_by_id(project_id) tmp_folder = self.file_helper.get_project_folder(project, self.file_helper.TEMP_FOLDER) result = OperationService().initiate_operation(current_user, project.id, adapter_instance, tmp_folder, visible, **data) self.logger.info("Finished operation:" + operation_name) return result except TVBException, excep: self.logger.exception("Could not launch operation " + operation_name + " with the given set of input data, because: " + excep.message) raise OperationException(excep.message, excep) except Exception, excep: self.logger.exception("Could not launch operation " + operation_name + " with the given set of input data!") raise OperationException(str(excep))
class SimulatorService(object): def __init__(self): self.logger = get_logger(self.__class__.__module__) self.burst_service = BurstService() self.operation_service = OperationService() self.files_helper = FilesHelper() @transactional def _prepare_operation(self, project_id, user_id, simulator_id, simulator_gid, algo_category, op_group, metadata, ranges=None): operation_parameters = json.dumps({'gid': simulator_gid.hex}) metadata, user_group = self.operation_service._prepare_metadata( metadata, algo_category, op_group, {}) meta_str = json.dumps(metadata) op_group_id = None if op_group: op_group_id = op_group.id operation = Operation(user_id, project_id, simulator_id, operation_parameters, op_group_id=op_group_id, meta=meta_str, range_values=ranges) self.logger.info("Saving Operation(userId=" + str(user_id) + ", projectId=" + str(project_id) + "," + str(metadata) + ", algorithmId=" + str(simulator_id) + ", ops_group= " + str(op_group_id) + ", params=" + str(operation_parameters) + ")") operation = dao.store_entity(operation) # TODO: prepare portlets/handle operation groups/no workflows return operation @staticmethod def _set_simulator_range_parameter(simulator, range_parameter_name, range_parameter_value): range_param_name_list = range_parameter_name.split('.') current_attr = simulator for param_name in range_param_name_list[:len(range_param_name_list) - 1]: current_attr = getattr(current_attr, param_name) setattr(current_attr, range_param_name_list[-1], range_parameter_value) def async_launch_and_prepare_simulation(self, burst_config, user, project, simulator_algo, session_stored_simulator, simulation_state_gid): try: metadata = {} metadata.update({DataTypeMetaData.KEY_BURST: burst_config.id}) simulator_id = simulator_algo.id algo_category = simulator_algo.algorithm_category operation = self._prepare_operation(project.id, user.id, simulator_id, session_stored_simulator.gid, algo_category, None, metadata) storage_path = self.files_helper.get_project_folder( project, str(operation.id)) SimulatorSerializer().serialize_simulator(session_stored_simulator, simulation_state_gid, storage_path) burst_config = self.burst_service.update_simulation_fields( burst_config.id, operation.id, session_stored_simulator.gid) self.burst_service.store_burst_configuration( burst_config, storage_path) wf_errs = 0 try: OperationService().launch_operation(operation.id, True) return operation except Exception as excep: self.logger.error(excep) wf_errs += 1 if burst_config: self.burst_service.mark_burst_finished( burst_config, error_message=str(excep)) self.logger.debug( "Finished launching workflow. The operation was launched successfully, " + str(wf_errs) + " had error on pre-launch steps") except Exception as excep: self.logger.error(excep) if burst_config: self.burst_service.mark_burst_finished( burst_config, error_message=str(excep)) def prepare_simulation_on_server(self, user_id, project, algorithm, zip_folder_path, simulator_file): with SimulatorH5(simulator_file) as simulator_h5: simulator_gid = simulator_h5.gid.load() metadata = {} simulator_id = algorithm.id algo_category = algorithm.algorithm_category operation = self._prepare_operation(project.id, user_id, simulator_id, simulator_gid, algo_category, None, metadata) storage_operation_path = self.files_helper.get_project_folder( project, str(operation.id)) self.async_launch_simulation_on_server(operation, zip_folder_path, storage_operation_path) return operation def async_launch_simulation_on_server(self, operation, zip_folder_path, storage_operation_path): try: for file in os.listdir(zip_folder_path): shutil.move(os.path.join(zip_folder_path, file), storage_operation_path) try: OperationService().launch_operation(operation.id, True) shutil.rmtree(zip_folder_path) return operation except Exception as excep: self.logger.error(excep) except Exception as excep: self.logger.error(excep) @staticmethod def _set_range_param_in_dict(param_value): if type(param_value) is numpy.ndarray: return param_value[0] elif isinstance(param_value, uuid.UUID): return param_value.hex else: return param_value def async_launch_and_prepare_pse(self, burst_config, user, project, simulator_algo, range_param1, range_param2, session_stored_simulator): try: simulator_id = simulator_algo.id algo_category = simulator_algo.algorithm_category operation_group = burst_config.operation_group metric_operation_group = burst_config.metric_operation_group operations = [] range_param2_values = [None] if range_param2: range_param2_values = range_param2.get_range_values() first_simulator = None for param1_value in range_param1.get_range_values(): for param2_value in range_param2_values: # Copy, but generate a new GUID for every Simulator in PSE simulator = copy.deepcopy(session_stored_simulator) simulator.gid = uuid.uuid4() self._set_simulator_range_parameter( simulator, range_param1.name, param1_value) ranges = { range_param1.name: self._set_range_param_in_dict(param1_value) } if param2_value is not None: self._set_simulator_range_parameter( simulator, range_param2.name, param2_value) ranges[ range_param2.name] = self._set_range_param_in_dict( param2_value) ranges = json.dumps(ranges) operation = self._prepare_operation( project.id, user.id, simulator_id, simulator.gid, algo_category, operation_group, {DataTypeMetaData.KEY_BURST: burst_config.id}, ranges) storage_path = self.files_helper.get_project_folder( project, str(operation.id)) SimulatorSerializer().serialize_simulator( simulator, None, storage_path) operations.append(operation) if first_simulator is None: first_simulator = simulator first_operation = operations[0] storage_path = self.files_helper.get_project_folder( project, str(first_operation.id)) burst_config = self.burst_service.update_simulation_fields( burst_config.id, first_operation.id, first_simulator.gid) self.burst_service.store_burst_configuration( burst_config, storage_path) datatype_group = DataTypeGroup( operation_group, operation_id=first_operation.id, fk_parent_burst=burst_config.id, state=json.loads( first_operation.meta_data)[DataTypeMetaData.KEY_STATE]) dao.store_entity(datatype_group) metrics_datatype_group = DataTypeGroup( metric_operation_group, fk_parent_burst=burst_config.id) dao.store_entity(metrics_datatype_group) wf_errs = 0 for operation in operations: try: OperationService().launch_operation(operation.id, True) except Exception as excep: self.logger.error(excep) wf_errs += 1 self.burst_service.mark_burst_finished( burst_config, error_message=str(excep)) self.logger.debug("Finished launching workflows. " + str(len(operations) - wf_errs) + " were launched successfully, " + str(wf_errs) + " had error on pre-launch steps") except Exception as excep: self.logger.error(excep) self.burst_service.mark_burst_finished(burst_config, error_message=str(excep)) def load_from_zip(self, zip_file, project): import_service = ImportService() simulator_folder = import_service.import_simulator_configuration_zip( zip_file) simulator_h5_filename = DirLoader( simulator_folder, None).find_file_for_has_traits_type(Simulator) with SimulatorH5(os.path.join(simulator_folder, simulator_h5_filename)) as sim_h5: simulator_gid = sim_h5.gid.load() simulator = SimulatorSerializer.deserialize_simulator( simulator_gid, simulator_folder) burst_config = self.burst_service.load_burst_configuration_from_folder( simulator_folder, project) return simulator, burst_config
class TestOperationResource(TransactionalTestCase): def transactional_setup_method(self): self.test_user = TestFactory.create_user('Rest_User') self.test_project = TestFactory.create_project(self.test_user, 'Rest_Project') self.operations_resource = GetOperationsInProjectResource() self.status_resource = GetOperationStatusResource() self.results_resource = GetOperationResultsResource() self.launch_resource = LaunchOperationResource() self.files_helper = FilesHelper() def test_server_get_operation_status_inexistent_gid(self): operation_gid = "inexistent-gid" with pytest.raises(InvalidIdentifierException): self.status_resource.get(operation_gid) def test_server_get_operation_status(self): zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_96.zip') TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path) operations = self.operations_resource.get(self.test_project.gid) result = self.status_resource.get(operations[0].gid) assert type(result) is str assert result in OperationPossibleStatus def test_server_get_operation_results_inexistent_gid(self): operation_gid = "inexistent-gid" with pytest.raises(InvalidIdentifierException): self.results_resource.get(operation_gid) def test_server_get_operation_results(self): zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_96.zip') TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path) operations = self.operations_resource.get(self.test_project.gid) result = self.results_resource.get(operations[0].gid) assert type(result) is list assert len(result) == 1 def test_server_get_operation_results_failed_operation(self): zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_90.zip') with pytest.raises(TVBException): TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path) operations = self.operations_resource.get(self.test_project.gid) result = self.results_resource.get(operations[0].gid) assert type(result) is list assert len(result) == 0 def test_server_launch_operation_no_file(self, mocker): # Mock flask.request.files to return a dictionary request_mock = mocker.patch.object(flask, 'request') request_mock.files = {} with pytest.raises(BadRequestException): self.launch_resource.post('', '', '') def test_server_launch_operation_wrong_file_extension(self, mocker): dummy_file = FileStorage(BytesIO(b"test"), 'test.txt') # Mock flask.request.files to return a dictionary request_mock = mocker.patch.object(flask, 'request') request_mock.files = {'file': dummy_file} with pytest.raises(BadRequestException): self.launch_resource.post('', '', '') def test_server_launch_operation_inexistent_gid(self, mocker): project_gid = "inexistent-gid" dummy_file = FileStorage(BytesIO(b"test"), 'test.h5') # Mock flask.request.files to return a dictionary request_mock = mocker.patch.object(flask, 'request') request_mock.files = {'file': dummy_file} with pytest.raises(InvalidIdentifierException): self.launch_resource.post(project_gid, '', '') def test_server_launch_operation_inexistent_algorithm(self, mocker): inexistent_algorithm = "inexistent-algorithm" dummy_file = FileStorage(BytesIO(b"test"), 'test.h5') # Mock flask.request.files to return a dictionary request_mock = mocker.patch.object(flask, 'request') request_mock.files = {'file': dummy_file} with pytest.raises(InvalidIdentifierException): self.launch_resource.post(self.test_project.gid, inexistent_algorithm, '') def test_server_launch_operation(self, mocker, time_series_index_factory): algorithm_module = "tvb.adapters.analyzers.fourier_adapter" algorithm_class = "FourierAdapter" input_ts_index = time_series_index_factory() fft_model = FFTAdapterModel() fft_model.time_series = UUID(input_ts_index.gid) fft_model.window_function = list(SUPPORTED_WINDOWING_FUNCTIONS)[0] input_folder = self.files_helper.get_project_folder(self.test_project) view_model_h5_path = h5.path_for(input_folder, ViewModelH5, fft_model.gid) view_model_h5 = ViewModelH5(view_model_h5_path, fft_model) view_model_h5.store(fft_model) view_model_h5.close() # Mock flask.request.files to return a dictionary request_mock = mocker.patch.object(flask, 'request') fp = open(view_model_h5_path, 'rb') request_mock.files = { 'file': FileStorage(fp, os.path.basename(view_model_h5_path)) } # Mock launch_operation() call mocker.patch.object(OperationService, 'launch_operation') operation_gid, status = self.launch_resource.post( self.test_project.gid, algorithm_module, algorithm_class) fp.close() assert type(operation_gid) is str assert len(operation_gid) > 0 def transactional_teardown_method(self): self.files_helper.remove_project_structure(self.test_project.name)
def tearDown(self): """ Clean-up tests data """ FilesHelper().remove_project_structure(self.test_project.name)
def teardown_method(self): """ Clean-up tests data """ self.clean_database() FilesHelper().remove_project_structure(self.test_project.name)
class FilesHelperTest(TransactionalTestCase): """ This class contains tests for the tvb.core.entities.file.files_helper module. """ PROJECT_NAME = "test_proj" def setUp(self): """ Set up the context needed by the tests. """ self.files_helper = FilesHelper() self.test_user = TestFactory.create_user() self.test_project = TestFactory.create_project(self.test_user, self.PROJECT_NAME) def tearDown(self): """ Remove generated project during tests. """ self.delete_project_folders() def test_check_created(self): """ Test standard flows for check created. """ self.files_helper.check_created() self.assertTrue(os.path.exists(root_storage), "Storage not created!") self.files_helper.check_created(os.path.join(root_storage, "test")) self.assertTrue(os.path.exists(root_storage), "Storage not created!") self.assertTrue(os.path.exists(os.path.join(root_storage, "test")), "Test directory not created!") def test_get_project_folder(self): """ Test the get_project_folder method which should create a folder in case it doesn't already exist. """ project_path = self.files_helper.get_project_folder(self.test_project) self.assertTrue(os.path.exists(project_path), "Folder doesn't exist") folder_path = self.files_helper.get_project_folder( self.test_project, "43") self.assertTrue(os.path.exists(project_path), "Folder doesn't exist") self.assertTrue(os.path.exists(folder_path), "Folder doesn't exist") def test_rename_project_structure(self): """ Try to rename the folder structure of a project. Standard flow. """ self.files_helper.get_project_folder(self.test_project) path, name = self.files_helper.rename_project_structure( self.test_project.name, "new_name") self.assertNotEqual(path, name, "Rename didn't take effect.") def test_rename_structure_same_name(self): """ Try to rename the folder structure of a project. Same name. """ self.files_helper.get_project_folder(self.test_project) self.assertRaises(FileStructureException, self.files_helper.rename_project_structure, self.test_project.name, self.PROJECT_NAME) def test_remove_project_structure(self): """ Check that remove project structure deletes the corresponding folder. Standard flow. """ full_path = self.files_helper.get_project_folder(self.test_project) self.assertTrue(os.path.exists(full_path), "Folder was not created.") self.files_helper.remove_project_structure(self.test_project.name) self.assertFalse(os.path.exists(full_path), "Project folder not deleted.") def test_write_project_metadata(self): """ Write XML for test-project. """ self.files_helper.write_project_metadata(self.test_project) expected_file = self.files_helper.get_project_meta_file_path( self.PROJECT_NAME) self.assertTrue(os.path.exists(expected_file)) project_meta = XMLReader(expected_file).read_metadata() loaded_project = model.Project(None, None) loaded_project.from_dict(project_meta, self.test_user.id) self.assertEqual(self.test_project.name, loaded_project.name) self.assertEqual(self.test_project.description, loaded_project.description) self.assertEqual(self.test_project.gid, loaded_project.gid) expected_dict = self.test_project.to_dict()[1] del expected_dict['last_updated'] found_dict = loaded_project.to_dict()[1] del found_dict['last_updated'] self.assertDictContainsSubset(expected_dict, found_dict) self.assertDictContainsSubset(found_dict, expected_dict) def test_write_operation_metadata(self): """ Test that a correct XML is created for an operation. """ operation = TestFactory.create_operation( test_user=self.test_user, test_project=self.test_project) expected_file = self.files_helper.get_operation_meta_file_path( self.PROJECT_NAME, operation.id) self.assertFalse(os.path.exists(expected_file)) self.files_helper.write_operation_metadata(operation) self.assertTrue(os.path.exists(expected_file)) operation_meta = XMLReader(expected_file).read_metadata() loaded_operation = model.Operation(None, None, None, None) loaded_operation.from_dict(operation_meta, dao) expected_dict = operation.to_dict()[1] found_dict = loaded_operation.to_dict()[1] for key, value in expected_dict.iteritems(): self.assertEqual(str(value), str(found_dict[key])) # Now validate that operation metaData can be also updated self.assertNotEqual("new_group_name", found_dict['user_group']) self.files_helper.update_operation_metadata(self.PROJECT_NAME, "new_group_name", operation.id) found_dict = XMLReader(expected_file).read_metadata() self.assertEqual("new_group_name", found_dict['user_group']) def test_remove_dt_happy_flow(self): """ Happy flow for removing a file related to a DataType. """ folder_path = self.files_helper.get_project_folder( self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path open(datatype.get_storage_file_path(), 'w') self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") self.files_helper.remove_datatype(datatype) self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not deleted!") def test_remove_dt_non_existent(self): """ Try to call remove on a dataType with no H5 file. Should work. """ folder_path = self.files_helper.get_project_folder( self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path self.assertFalse(os.path.exists(datatype.get_storage_file_path())) self.files_helper.remove_datatype(datatype) def test_move_datatype(self): """ Make sure associated H5 file is moved to a correct new location. """ folder_path = self.files_helper.get_project_folder( self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path open(datatype.get_storage_file_path(), 'w') self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") self.files_helper.move_datatype(datatype, self.PROJECT_NAME + '11', "43") self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not moved!") datatype.storage_path = self.files_helper.get_project_folder( self.PROJECT_NAME + '11', "43") self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") def test_find_relative_path(self): """ Tests that relative path is computed properly. """ rel_path = self.files_helper.find_relative_path( "/root/up/to/here/test/it/now", "/root/up/to/here") self.assertEqual(rel_path, os.sep.join(["test", "it", "now"]), "Did not extract relative path as expected.") def test_remove_files_valid(self): """ Pass a valid list of files and check they are all removed. """ file_list = ["test1", "test2", "test3"] for file_n in file_list: fp = open(file_n, 'w') fp.write('test') fp.close() for file_n in file_list: self.assertTrue(os.path.isfile(file_n)) self.files_helper.remove_files(file_list) for file_n in file_list: self.assertFalse(os.path.isfile(file_n)) def test_remove_folder(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" os.mkdir(folder_name) self.assertTrue(os.path.isdir(folder_name), "Folder should be created.") self.files_helper.remove_folder(folder_name) self.assertFalse(os.path.isdir(folder_name), "Folder should be deleted.") def test_remove_folder_non_existing_ignore_exc(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.") self.files_helper.remove_folder(folder_name, ignore_errors=True) def test_remove_folder_non_existing(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.") self.assertRaises(FileStructureException, self.files_helper.remove_folder, folder_name, False)
class ProjectService: """ Services layer for Project entities. """ def __init__(self): self.logger = get_logger(__name__) self.structure_helper = FilesHelper() def store_project(self, current_user, is_create, selected_id, **data): """ We want to create/update a project entity. """ # Validate Unique Name new_name = data["name"] if len(new_name) < 1: raise ProjectServiceException("Invalid project name!") projects_no = dao.count_projects_for_name(new_name, selected_id) if projects_no > 0: err = {'name': 'Please choose another name, this one is used!'} raise formencode.Invalid("Duplicate Name Error", {}, None, error_dict=err) started_operations = dao.get_operation_numbers(selected_id)[1] if started_operations > 0: raise ProjectServiceException( "A project can not be renamed while operations are still running!" ) if is_create: current_proj = Project(new_name, current_user.id, data["description"]) self.structure_helper.get_project_folder(current_proj) else: try: current_proj = dao.get_project_by_id(selected_id) except Exception as excep: self.logger.exception("An error has occurred!") raise ProjectServiceException(str(excep)) if current_proj.name != new_name: self.structure_helper.rename_project_structure( current_proj.name, new_name) current_proj.name = new_name current_proj.description = data["description"] # Commit to make sure we have a valid ID current_proj.refresh_update_date() self.structure_helper.write_project_metadata(current_proj) current_proj = dao.store_entity(current_proj) # Retrieve, to initialize lazy attributes current_proj = dao.get_project_by_id(current_proj.id) # Update share settings on current Project entity visited_pages = [] prj_admin = current_proj.administrator.username if 'visited_pages' in data and data['visited_pages']: visited_pages = data['visited_pages'].split(',') for page in visited_pages: members = UserService.retrieve_users_except([prj_admin], int(page), MEMBERS_PAGE_SIZE)[0] members = [m.id for m in members] dao.delete_members_for_project(current_proj.id, members) selected_user_ids = data["users"] if is_create and current_user.id not in selected_user_ids: # Make the project admin also member of the current project selected_user_ids.append(current_user.id) dao.add_members_to_project(current_proj.id, selected_user_ids) # Finish operation self.logger.debug("Edit/Save OK for project:" + str(current_proj.id) + ' by user:'******'-' result["count"] = one_op[2] result["gid"] = one_op[13] if one_op[3] is not None and one_op[3]: try: operation_group = dao.get_generic_entity( OperationGroup, one_op[3])[0] result["group"] = operation_group.name result["group"] = result["group"].replace("_", " ") result["operation_group_id"] = operation_group.id datatype_group = dao.get_datatypegroup_by_op_group_id( one_op[3]) result["datatype_group_gid"] = datatype_group.gid result["gid"] = operation_group.gid # Filter only viewers for current DataTypeGroup entity: result["view_groups"] = AlgorithmService( ).get_visualizers_for_group(datatype_group.gid) except Exception: self.logger.exception( "We will ignore group on entity:" + str(one_op)) result["datatype_group_gid"] = None else: result['group'] = None result['datatype_group_gid'] = None result["algorithm"] = dao.get_algorithm_by_id(one_op[4]) result["user"] = dao.get_user_by_id(one_op[5]) if type(one_op[6]) is str: result["create"] = string2date(str(one_op[6])) else: result["create"] = one_op[6] if type(one_op[7]) is str: result["start"] = string2date(str(one_op[7])) else: result["start"] = one_op[7] if type(one_op[8]) is str: result["complete"] = string2date(str(one_op[8])) else: result["complete"] = one_op[8] if result["complete"] is not None and result[ "start"] is not None: result["duration"] = format_timedelta(result["complete"] - result["start"]) result["status"] = one_op[9] result["additional"] = one_op[10] result["visible"] = True if one_op[11] > 0 else False result['operation_tag'] = one_op[12] result['figures'] = None if not result['group']: datatype_results = dao.get_results_for_operation( result['id']) result['results'] = [] for dt in datatype_results: dt_loaded = ABCAdapter.load_entity_by_gid(dt.gid) if dt_loaded: result['results'].append(dt_loaded) else: self.logger.warning( "Could not retrieve datatype %s" % str(dt)) operation_figures = dao.get_figures_for_operation( result['id']) # Compute the full path to the figure / image on disk for figure in operation_figures: figures_folder = self.structure_helper.get_images_folder( figure.project.name) figure_full_path = os.path.join( figures_folder, figure.file_path) # Compute the path available from browser figure.figure_path = utils.path2url_part( figure_full_path) result['figures'] = operation_figures else: result['results'] = None operations.append(result) except Exception: # We got an exception when processing one Operation Row. We will continue with the rest of the rows. self.logger.exception( "Could not prepare operation for display:" + str(one_op)) return selected_project, total_ops_nr, operations, pages_no def retrieve_projects_for_user(self, user_id, current_page=1): """ Return a list with all Projects visible for current user. """ start_idx = PROJECTS_PAGE_SIZE * (current_page - 1) total = dao.get_projects_for_user(user_id, is_count=True) available_projects = dao.get_projects_for_user(user_id, start_idx, PROJECTS_PAGE_SIZE) pages_no = total // PROJECTS_PAGE_SIZE + (1 if total % PROJECTS_PAGE_SIZE else 0) for prj in available_projects: fns, sta, err, canceled, pending = dao.get_operation_numbers( prj.id) prj.operations_finished = fns prj.operations_started = sta prj.operations_error = err prj.operations_canceled = canceled prj.operations_pending = pending prj.disk_size = dao.get_project_disk_size(prj.id) prj.disk_size_human = format_bytes_human(prj.disk_size) self.logger.debug("Displaying " + str(len(available_projects)) + " projects in UI for user " + str(user_id)) return available_projects, pages_no @staticmethod def retrieve_all_user_projects(user_id, page_start=0, page_size=PROJECTS_PAGE_SIZE): """ Return a list with all projects visible for current user, without pagination. """ return dao.get_projects_for_user(user_id, page_start=page_start, page_size=page_size) @staticmethod def get_linkable_projects_for_user(user_id, data_id): """ Find projects with are visible for current user, and in which current datatype hasn't been linked yet. """ return dao.get_linkable_projects_for_user(user_id, data_id) @transactional def remove_project(self, project_id): """ Remove Project from DB and File Storage. """ try: project2delete = dao.get_project_by_id(project_id) self.logger.debug("Deleting project: id=" + str(project_id) + ' name=' + project2delete.name) project_bursts = dao.get_bursts_for_project(project_id) for burst in project_bursts: dao.remove_entity(burst.__class__, burst.id) project_datatypes = dao.get_datatypes_in_project(project_id) for one_data in project_datatypes: self.remove_datatype(project_id, one_data.gid, True) links = dao.get_links_for_project(project_id) for one_link in links: dao.remove_entity(Links, one_link.id) self.structure_helper.remove_project_structure(project2delete.name) dao.delete_project(project_id) self.logger.debug("Deleted project: id=" + str(project_id) + ' name=' + project2delete.name) except RemoveDataTypeException as excep: self.logger.exception("Could not execute operation Node Remove!") raise ProjectServiceException(str(excep)) except FileStructureException as excep: self.logger.exception("Could not delete because of rights!") raise ProjectServiceException(str(excep)) except Exception as excep: self.logger.exception(str(excep)) raise ProjectServiceException(str(excep)) # ----------------- Methods for populating Data-Structure Page --------------- @staticmethod def get_datatype_in_group(group): """ Return all dataTypes that are the result of the same DTgroup. """ return dao.get_datatype_in_group(datatype_group_id=group) @staticmethod def get_datatypes_from_datatype_group(datatype_group_id): """ Retrieve all dataType which are part from the given dataType group. """ return dao.get_datatypes_from_datatype_group(datatype_group_id) @staticmethod def load_operation_by_gid(operation_gid): """ Retrieve loaded Operation from DB""" return dao.get_operation_by_gid(operation_gid) @staticmethod def load_operation_lazy_by_gid(operation_gid): """ Retrieve lazy Operation from DB""" return dao.get_operation_lazy_by_gid(operation_gid) @staticmethod def get_operation_group_by_id(operation_group_id): """ Loads OperationGroup from DB""" return dao.get_operationgroup_by_id(operation_group_id) @staticmethod def get_operation_group_by_gid(operation_group_gid): """ Loads OperationGroup from DB""" return dao.get_operationgroup_by_gid(operation_group_gid) @staticmethod def get_operations_in_group(operation_group): """ Return all the operations from an operation group. """ return dao.get_operations_in_group(operation_group.id) @staticmethod def is_upload_operation(operation_gid): """ Returns True only if the operation with the given GID is an upload operation. """ return dao.is_upload_operation(operation_gid) @staticmethod def get_all_operations_for_uploaders(project_id): """ Returns all finished upload operations. """ return dao.get_all_operations_for_uploaders(project_id) def set_operation_and_group_visibility(self, entity_gid, is_visible, is_operation_group=False): """ Sets the operation visibility. If 'is_operation_group' is True than this method will change the visibility for all the operation from the OperationGroup with the GID field equal to 'entity_gid'. """ def set_visibility(op): # workaround: # 'reload' the operation so that it has the project property set. # get_operations_in_group does not eager load it and now we're out of a sqlalchemy session # write_operation_metadata requires that property op = dao.get_operation_by_id(op.id) # end hack op.visible = is_visible dao.store_entity(op) def set_group_descendants_visibility(operation_group_id): ops_in_group = dao.get_operations_in_group(operation_group_id) for group_op in ops_in_group: set_visibility(group_op) if is_operation_group: op_group_id = dao.get_operationgroup_by_gid(entity_gid).id set_group_descendants_visibility(op_group_id) else: operation = dao.get_operation_by_gid(entity_gid) # we assure that if the operation belongs to a group than the visibility will be changed for the entire group if operation.fk_operation_group is not None: set_group_descendants_visibility(operation.fk_operation_group) else: set_visibility(operation) def get_operation_details(self, operation_gid, is_group): """ :returns: an entity OperationOverlayDetails filled with all information for current operation details. """ if is_group: operation_group = self.get_operation_group_by_gid(operation_gid) operation = dao.get_operations_in_group(operation_group.id, False, True) # Reload, to make sure all attributes lazy are populated as well. operation = dao.get_operation_by_gid(operation.gid) no_of_op_in_group = dao.get_operations_in_group(operation_group.id, is_count=True) datatype_group = self.get_datatypegroup_by_op_group_id( operation_group.id) count_result = dao.count_datatypes_in_group(datatype_group.id) else: operation = dao.get_operation_by_gid(operation_gid) if operation is None: return None no_of_op_in_group = 1 count_result = dao.count_resulted_datatypes(operation.id) user_display_name = dao.get_user_by_id( operation.fk_launched_by).display_name burst = dao.get_burst_for_operation_id(operation.id) datatypes_param, all_special_params = self._review_operation_inputs( operation.gid) op_pid = dao.get_operation_process_for_operation(operation.id) op_details = OperationOverlayDetails(operation, user_display_name, len(datatypes_param), count_result, burst, no_of_op_in_group, op_pid) # Add all parameter which are set differently by the user on this Operation. if all_special_params is not None: op_details.add_scientific_fields(all_special_params) return op_details @staticmethod def get_filterable_meta(): """ Contains all the attributes by which the user can structure the tree of DataTypes """ return DataTypeMetaData.get_filterable_meta() def get_project_structure(self, project, visibility_filter, first_level, second_level, filter_value): """ Find all DataTypes (including the linked ones and the groups) relevant for the current project. In case of a problem, will return an empty list. """ metadata_list = [] dt_list = dao.get_data_in_project(project.id, visibility_filter, filter_value) for dt in dt_list: # Prepare the DT results from DB, for usage in controller, by converting into DataTypeMetaData objects data = {} is_group = False group_op = None dt_entity = dao.get_datatype_by_gid(dt.gid) if dt_entity is None: self.logger.warning( "Ignored entity (possibly removed DT class)" + str(dt)) continue # Filter by dt.type, otherwise Links to individual DT inside a group will be mistaken if dt.type == "DataTypeGroup" and dt.parent_operation.operation_group is not None: is_group = True group_op = dt.parent_operation.operation_group # All these fields are necessary here for dynamic Tree levels. data[DataTypeMetaData.KEY_DATATYPE_ID] = dt.id data[DataTypeMetaData.KEY_GID] = dt.gid data[DataTypeMetaData.KEY_NODE_TYPE] = dt.display_type data[DataTypeMetaData.KEY_STATE] = dt.state data[DataTypeMetaData.KEY_SUBJECT] = str(dt.subject) data[DataTypeMetaData.KEY_TITLE] = dt_entity.display_name data[DataTypeMetaData.KEY_RELEVANCY] = dt.visible data[DataTypeMetaData. KEY_LINK] = dt.parent_operation.fk_launched_in != project.id data[DataTypeMetaData. KEY_TAG_1] = dt.user_tag_1 if dt.user_tag_1 else '' data[DataTypeMetaData. KEY_TAG_2] = dt.user_tag_2 if dt.user_tag_2 else '' data[DataTypeMetaData. KEY_TAG_3] = dt.user_tag_3 if dt.user_tag_3 else '' data[DataTypeMetaData. KEY_TAG_4] = dt.user_tag_4 if dt.user_tag_4 else '' data[DataTypeMetaData. KEY_TAG_5] = dt.user_tag_5 if dt.user_tag_5 else '' # Operation related fields: operation_name = CommonDetails.compute_operation_name( dt.parent_operation.algorithm.algorithm_category.displayname, dt.parent_operation.algorithm.displayname) data[DataTypeMetaData.KEY_OPERATION_TYPE] = operation_name data[ DataTypeMetaData. KEY_OPERATION_ALGORITHM] = dt.parent_operation.algorithm.displayname data[DataTypeMetaData. KEY_AUTHOR] = dt.parent_operation.user.username data[ DataTypeMetaData. KEY_OPERATION_TAG] = group_op.name if is_group else dt.parent_operation.user_group data[DataTypeMetaData. KEY_OP_GROUP_ID] = group_op.id if is_group else None completion_date = dt.parent_operation.completion_date string_year = completion_date.strftime( MONTH_YEAR_FORMAT) if completion_date is not None else "" string_month = completion_date.strftime( DAY_MONTH_YEAR_FORMAT) if completion_date is not None else "" data[DataTypeMetaData.KEY_DATE] = date2string(completion_date) if ( completion_date is not None) else '' data[DataTypeMetaData.KEY_CREATE_DATA_MONTH] = string_year data[DataTypeMetaData.KEY_CREATE_DATA_DAY] = string_month data[ DataTypeMetaData. KEY_BURST] = dt._parent_burst.name if dt._parent_burst is not None else '-None-' metadata_list.append(DataTypeMetaData(data, dt.invalid)) return StructureNode.metadata2tree(metadata_list, first_level, second_level, project.id, project.name) @staticmethod def get_datatype_details(datatype_gid): """ :returns: an array. First entry in array is an instance of DataTypeOverlayDetails\ The second one contains all the possible states for the specified dataType. """ meta_atts = DataTypeOverlayDetails() states = DataTypeMetaData.STATES try: datatype_result = dao.get_datatype_details(datatype_gid) meta_atts.fill_from_datatype(datatype_result, datatype_result._parent_burst) return meta_atts, states, datatype_result except Exception: # We ignore exception here (it was logged above, and we want to return no details). return meta_atts, states, None def _remove_project_node_files(self, project_id, gid, skip_validation=False): """ Delegate removal of a node in the structure of the project. In case of a problem will THROW StructureException. """ try: project = self.find_project(project_id) datatype = dao.get_datatype_by_gid(gid) links = dao.get_links_for_datatype(datatype.id) if links: was_link = False for link in links: # This means it's only a link and we need to remove it if link.fk_from_datatype == datatype.id and link.fk_to_project == project.id: dao.remove_entity(Links, link.id) was_link = True if not was_link: # Create a clone of the operation new_op = Operation( dao.get_system_user().id, links[0].fk_to_project, datatype.parent_operation.fk_from_algo, datatype.parent_operation.parameters, datatype.parent_operation.status, datatype.parent_operation.start_date, datatype.parent_operation.completion_date, datatype.parent_operation.fk_operation_group, datatype.parent_operation.additional_info, datatype.parent_operation.user_group, datatype.parent_operation.range_values) new_op = dao.store_entity(new_op) to_project = self.find_project(links[0].fk_to_project).name full_path = h5.path_for_stored_index(datatype) self.structure_helper.move_datatype( datatype, to_project, str(new_op.id), full_path) datatype.fk_from_operation = new_op.id datatype.parent_operation = new_op dao.store_entity(datatype) dao.remove_entity(Links, links[0].id) else: specific_remover = get_remover(datatype.type)(datatype) specific_remover.remove_datatype(skip_validation) h5_path = h5.path_for_stored_index(datatype) self.structure_helper.remove_datatype_file(h5_path) except RemoveDataTypeException: self.logger.exception("Could not execute operation Node Remove!") raise except FileStructureException: self.logger.exception("Remove operation failed") raise StructureException( "Remove operation failed for unknown reasons.Please contact system administrator." ) def remove_operation(self, operation_id): """ Remove a given operation """ operation = dao.try_get_operation_by_id(operation_id) if operation is not None: self.logger.debug("Deleting operation %s " % operation) datatypes_for_op = dao.get_results_for_operation(operation_id) for dt in reversed(datatypes_for_op): self.remove_datatype(operation.project.id, dt.gid, False) # Here the Operation is mot probably already removed - in case DTs were found inside # but we still remove it for the case when no DTs exist dao.remove_entity(Operation, operation.id) self.structure_helper.remove_operation_data( operation.project.name, operation_id) self.logger.debug("Finished deleting operation %s " % operation) else: self.logger.warning( "Attempt to delete operation with id=%s which no longer exists." % operation_id) def remove_datatype(self, project_id, datatype_gid, skip_validation=False): """ Method used for removing a dataType. If the given dataType is a DatatypeGroup or a dataType from a DataTypeGroup than this method will remove the entire group. The operation(s) used for creating the dataType(s) will also be removed. """ datatype = dao.get_datatype_by_gid(datatype_gid) if datatype is None: self.logger.warning( "Attempt to delete DT[%s] which no longer exists." % datatype_gid) return is_datatype_group = False if dao.is_datatype_group(datatype_gid): is_datatype_group = True elif datatype.fk_datatype_group is not None: is_datatype_group = True datatype = dao.get_datatype_by_id(datatype.fk_datatype_group) operations_set = [datatype.fk_from_operation] correct = True if is_datatype_group: self.logger.debug("Removing datatype group %s" % datatype) data_list = dao.get_datatypes_from_datatype_group(datatype.id) for adata in data_list: self._remove_project_node_files(project_id, adata.gid, skip_validation) if adata.fk_from_operation not in operations_set: operations_set.append(adata.fk_from_operation) datatype_group = dao.get_datatype_group_by_gid(datatype.gid) dao.remove_entity(DataTypeGroup, datatype.id) correct = correct and dao.remove_entity( OperationGroup, datatype_group.fk_operation_group) else: self.logger.debug("Removing datatype %s" % datatype) self._remove_project_node_files(project_id, datatype.gid, skip_validation) # Remove Operation entity in case no other DataType needs them. project = dao.get_project_by_id(project_id) for operation_id in operations_set: dependent_dt = dao.get_generic_entity(DataType, operation_id, "fk_from_operation") if len(dependent_dt) > 0: # Do not remove Operation in case DataType still exist referring it. continue correct = correct and dao.remove_entity(Operation, operation_id) # Make sure Operation folder is removed self.structure_helper.remove_operation_data( project.name, datatype.fk_from_operation) if not correct: raise RemoveDataTypeException("Could not remove DataType " + str(datatype_gid)) def update_metadata(self, submit_data): """ Update DataType/ DataTypeGroup metadata THROW StructureException when input data is invalid. """ new_data = dict() for key in DataTypeOverlayDetails().meta_attributes_list: if key in submit_data: value = submit_data[key] if value == "None": value = None if value == "" and key in [ CommonDetails.CODE_OPERATION_TAG, CommonDetails.CODE_OPERATION_GROUP_ID ]: value = None new_data[key] = value try: if (CommonDetails.CODE_OPERATION_GROUP_ID in new_data and new_data[CommonDetails.CODE_OPERATION_GROUP_ID]): # We need to edit a group all_data_in_group = dao.get_datatype_in_group( operation_group_id=new_data[ CommonDetails.CODE_OPERATION_GROUP_ID]) if len(all_data_in_group) < 1: raise StructureException( "Inconsistent group, can not be updated!") # datatype_group = dao.get_generic_entity(DataTypeGroup, all_data_in_group[0].fk_datatype_group)[0] # all_data_in_group.append(datatype_group) for datatype in all_data_in_group: self._edit_data(datatype, new_data, True) else: # Get the required DataType and operation from DB to store changes that will be done in XML. gid = new_data[CommonDetails.CODE_GID] datatype = dao.get_datatype_by_gid(gid) self._edit_data(datatype, new_data) except Exception as excep: self.logger.exception(excep) raise StructureException(str(excep)) def _edit_data(self, datatype, new_data, from_group=False): # type: (DataType, dict, bool) -> None """ Private method, used for editing a meta-data XML file and a DataType row for a given custom DataType entity with new dictionary of data from UI. """ # 1. First update Operation fields: # Update group field if possible new_group_name = new_data[CommonDetails.CODE_OPERATION_TAG] empty_group_value = (new_group_name is None or new_group_name == "") if from_group: if empty_group_value: raise StructureException("Empty group is not allowed!") group = dao.get_generic_entity( OperationGroup, new_data[CommonDetails.CODE_OPERATION_GROUP_ID]) if group and len(group) > 0 and new_group_name != group[0].name: group = group[0] exists_group = dao.get_generic_entity(OperationGroup, new_group_name, 'name') if exists_group: raise StructureException("Group '" + new_group_name + "' already exists.") group.name = new_group_name dao.store_entity(group) else: operation = dao.get_operation_by_id(datatype.fk_from_operation) operation.user_group = new_group_name dao.store_entity(operation) op_folder = self.structure_helper.get_project_folder( operation.project, str(operation.id)) vm_gid = json.loads(operation.parameters)['gid'] view_model_file = h5.determine_filepath(vm_gid, op_folder) if view_model_file: view_model_class = H5File.determine_type(view_model_file) view_model = view_model_class() with ViewModelH5(view_model_file, view_model) as f: ga = f.load_generic_attributes() ga.operation_tag = new_group_name f.store_generic_attributes(ga, False) else: self.logger.warning( "Could not find ViewModel H5 file for op: {}".format( operation)) # 2. Update GenericAttributes in the associated H5 files: h5_path = h5.path_for_stored_index(datatype) with H5File.from_file(h5_path) as f: ga = f.load_generic_attributes() ga.subject = new_data[DataTypeOverlayDetails.DATA_SUBJECT] ga.state = new_data[DataTypeOverlayDetails.DATA_STATE] ga.operation_tag = new_group_name if DataTypeOverlayDetails.DATA_TAG_1 in new_data: ga.user_tag_1 = new_data[DataTypeOverlayDetails.DATA_TAG_1] if DataTypeOverlayDetails.DATA_TAG_2 in new_data: ga.user_tag_2 = new_data[DataTypeOverlayDetails.DATA_TAG_2] if DataTypeOverlayDetails.DATA_TAG_3 in new_data: ga.user_tag_3 = new_data[DataTypeOverlayDetails.DATA_TAG_3] if DataTypeOverlayDetails.DATA_TAG_4 in new_data: ga.user_tag_4 = new_data[DataTypeOverlayDetails.DATA_TAG_4] if DataTypeOverlayDetails.DATA_TAG_5 in new_data: ga.user_tag_5 = new_data[DataTypeOverlayDetails.DATA_TAG_5] f.store_generic_attributes(ga, False) # 3. Update MetaData in DT Index DB as well. datatype.fill_from_generic_attributes(ga) dao.store_entity(datatype) def get_datatype_and_datatypegroup_inputs_for_operation( self, operation_gid, selected_filter): """ Returns the dataTypes that are used as input parameters for the given operation. 'selected_filter' - is expected to be a visibility filter. If any dataType is part of a dataType group then the dataType group will be returned instead of that dataType. """ all_datatypes = self._review_operation_inputs(operation_gid)[0] datatype_inputs = [] for datatype in all_datatypes: if selected_filter.display_name == StaticFiltersFactory.RELEVANT_VIEW: if datatype.visible: datatype_inputs.append(datatype) else: datatype_inputs.append(datatype) datatypes = [] datatype_groups = dict() for data_type in datatype_inputs: if data_type.fk_datatype_group is None: datatypes.append(data_type) elif data_type.fk_datatype_group not in datatype_groups: dt_group = dao.get_datatype_by_id(data_type.fk_datatype_group) datatype_groups[data_type.fk_datatype_group] = dt_group datatypes.extend([v for v in datatype_groups.values()]) return datatypes def _review_operation_inputs(self, operation_gid): """ :returns: A list of DataTypes that are used as input parameters for the specified operation. And a dictionary will all operation parameters different then the default ones. """ operation = dao.get_operation_by_gid(operation_gid) try: adapter = ABCAdapter.build_adapter(operation.algorithm) return review_operation_inputs_from_adapter(adapter, operation) except Exception: self.logger.exception("Could not load details for operation %s" % operation_gid) parameters = json.loads(operation.parameters) if 'gid' in parameters.keys(): changed_parameters = dict( Warning= "Algorithm changed dramatically. We can not offer more details" ) else: changed_parameters = dict( Warning= "GID parameter is missing. Old implementation of the operation." ) return [], changed_parameters def get_datatypes_inputs_for_operation_group(self, group_id, selected_filter): """ Returns the dataType inputs for an operation group. If more dataTypes are part of the same dataType group then only the dataType group will be returned instead of them. """ operations_gids = dao.get_operations_in_group(group_id, only_gids=True) op_group_inputs = dict() for gid in operations_gids: op_inputs = self.get_datatype_and_datatypegroup_inputs_for_operation( gid[0], selected_filter) for datatype in op_inputs: op_group_inputs[datatype.id] = datatype return list(op_group_inputs.values()) @staticmethod def get_results_for_operation(operation_id, selected_filter=None): """ Retrieve the DataTypes entities resulted after the execution of the given operation. """ return dao.get_results_for_operation(operation_id, selected_filter) @staticmethod def get_operations_for_datatype_group(datatype_group_id, visibility_filter, only_in_groups=False): """ Returns all the operations which uses as an input parameter a dataType from the given DataTypeGroup. visibility_filter - is a filter used for retrieving all the operations or only the relevant ones. If only_in_groups is True than this method will return only the operations that are part from an operation group, otherwise it will return only the operations that are NOT part of an operation group. """ if visibility_filter.display_name != StaticFiltersFactory.RELEVANT_VIEW: return dao.get_operations_for_datatype_group( datatype_group_id, only_relevant=False, only_in_groups=only_in_groups) return dao.get_operations_for_datatype_group( datatype_group_id, only_in_groups=only_in_groups) @staticmethod def get_operations_for_datatype(datatype_gid, visibility_filter, only_in_groups=False): """ Returns all the operations which uses as an input parameter the dataType with the specified GID. If only_in_groups is True than this method will return only the operations that are part from an operation group, otherwise it will return only the operations that are NOT part of an operation group. """ if visibility_filter.display_name != StaticFiltersFactory.RELEVANT_VIEW: return dao.get_operations_for_datatype( datatype_gid, only_relevant=False, only_in_groups=only_in_groups) return dao.get_operations_for_datatype(datatype_gid, only_in_groups=only_in_groups) @staticmethod def get_datatype_by_id(datatype_id): """Retrieve a DataType DB reference by its id.""" return dao.get_datatype_by_id(datatype_id) @staticmethod def get_datatypegroup_by_gid(datatypegroup_gid): """ Returns the DataTypeGroup with the specified gid. """ return dao.get_datatype_group_by_gid(datatypegroup_gid) @staticmethod def count_datatypes_generated_from(datatype_gid): """ A list with all the datatypes resulted from operations that had as input the datatype given by 'datatype_gid'. """ return dao.count_datatypes_generated_from(datatype_gid) @staticmethod def get_datatypegroup_by_op_group_id(operation_group_id): """ Returns the DataTypeGroup with the specified id. """ return dao.get_datatypegroup_by_op_group_id(operation_group_id) @staticmethod def get_datatypes_in_project(project_id, only_visible=False): return dao.get_data_in_project(project_id, only_visible) @staticmethod def set_datatype_visibility(datatype_gid, is_visible): """ Sets the dataType visibility. If the given dataType is a dataType group or it is part of a dataType group than this method will set the visibility for each dataType from this group. """ def set_visibility(dt): """ set visibility flag, persist in db and h5""" dt.visible = is_visible dt = dao.store_entity(dt) h5_path = h5.path_for_stored_index(dt) with H5File.from_file(h5_path) as f: f.visible.store(is_visible) def set_group_descendants_visibility(datatype_group_id): datatypes_in_group = dao.get_datatypes_from_datatype_group( datatype_group_id) for group_dt in datatypes_in_group: set_visibility(group_dt) datatype = dao.get_datatype_by_gid(datatype_gid) if isinstance(datatype, DataTypeGroup): # datatype is a group set_group_descendants_visibility(datatype.id) datatype.visible = is_visible dao.store_entity(datatype) elif datatype.fk_datatype_group is not None: # datatype is member of a group set_group_descendants_visibility(datatype.fk_datatype_group) # the datatype to be updated is the parent datatype group parent = dao.get_datatype_by_id(datatype.fk_datatype_group) parent.visible = is_visible dao.store_entity(parent) else: # update the single datatype. set_visibility(datatype) @staticmethod def is_datatype_group(datatype_gid): """ Used to check if the dataType with the specified GID is a DataTypeGroup. """ return dao.is_datatype_group(datatype_gid)
class SimulatorController(BurstBaseController): KEY_IS_LOAD_AFTER_REDIRECT = "is_load_after_redirect" COPY_NAME_FORMAT = "copy_of_{}" BRANCH_NAME_FORMAT = "{}_branch{}" def __init__(self): BurstBaseController.__init__(self) self.range_parameters = SimulatorRangeParameters() self.burst_service = BurstService() self.simulator_service = SimulatorService() self.files_helper = FilesHelper() self.cached_simulator_algorithm = self.algorithm_service.get_algorithm_by_module_and_class( IntrospectionRegistry.SIMULATOR_MODULE, IntrospectionRegistry.SIMULATOR_CLASS) self.context = SimulatorContext() self.monitors_handler = MonitorsWizardHandler() @expose_json def cancel_or_remove_burst(self, burst_id): """ Cancel or Remove the burst entity given by burst_id (and all linked entities: op, DTs) :returns True: if the op was successfully. """ burst_config = BurstService.load_burst_configuration(int(burst_id)) op_id, is_group = burst_config.operation_info_for_burst_removal return self.cancel_or_remove_operation(op_id, is_group, burst_config.is_finished) def cancel_or_remove_operation(self, operation_id, is_group, remove_after_stop=False): """ Stop the operation given by operation_id. If is_group is true stop all the operations from that group. """ # Load before we remove, to have its data in memory here burst_config = BurstService.get_burst_for_operation_id( operation_id, is_group) result = OperationService.stop_operation(operation_id, is_group, remove_after_stop) if remove_after_stop: current_burst = self.context.burst_config if (current_burst is not None and burst_config is not None and current_burst.id == burst_config.id and ((current_burst.fk_simulation == operation_id and not is_group) or (current_burst.fk_operation_group == operation_id and is_group))): self.reset_simulator_configuration() return result @expose_page @settings @context_selected def index(self): """Get on burst main page""" template_specification = dict( mainContent="burst/main_burst", title="Simulation Cockpit", includedResources='project/included_resources') if not self.context.last_loaded_fragment_url: self.context.add_last_loaded_form_url_to_session( SimulatorWizzardURLs.SET_CONNECTIVITY_URL) self.context.set_burst_config() _, is_simulation_copy, is_simulation_load, is_branch = self.context.get_common_params( ) if self.context.burst_config.start_time is not None: is_simulation_load = True self.context.add_simulator_load_to_session(True) template_specification['burstConfig'] = self.context.burst_config template_specification[ 'burst_list'] = self.burst_service.get_available_bursts( self.context.project.id) portlets_list = [] # self.burst_service.get_available_portlets() template_specification['portletList'] = portlets_list template_specification['selectedPortlets'] = json.dumps(portlets_list) form = self.prepare_first_fragment() rendering_rules = SimulatorFragmentRenderingRules( form, SimulatorWizzardURLs.SET_CONNECTIVITY_URL, None, is_simulation_copy, is_simulation_load, last_form_url=self.context.last_loaded_fragment_url, last_request_type=cherrypy.request.method, is_first_fragment=True, is_branch=is_branch) template_specification.update(**rendering_rules.to_dict()) return self.fill_default_attributes(template_specification) def prepare_first_fragment(self): self.context.set_simulator() simulator, _, _, is_branch = self.context.get_common_params() branch_conditions = self.simulator_service.compute_conn_branch_conditions( is_branch, simulator) form = self.algorithm_service.prepare_adapter_form( form_instance=SimulatorAdapterForm(), project_id=self.context.project.id, extra_conditions=branch_conditions) self.simulator_service.validate_first_fragment(form, self.context.project.id, ConnectivityIndex) form.fill_from_trait(self.context.simulator) return form @expose_fragment('simulator_fragment') def set_connectivity(self, **data): session_stored_simulator, is_simulation_copy, is_simulation_load, _ = self.context.get_common_params( ) if cherrypy.request.method == POST_REQUEST: self.context.add_last_loaded_form_url_to_session( SimulatorWizzardURLs.SET_COUPLING_PARAMS_URL) form = SimulatorAdapterForm() form.fill_from_post(data) self.simulator_service.reset_at_connectivity_change( is_simulation_copy, form, session_stored_simulator) form.fill_trait(session_stored_simulator) next_form = self.algorithm_service.prepare_adapter_form( form_instance=get_form_for_coupling( type(session_stored_simulator.coupling))()) self.range_parameters.coupling_parameters = next_form.get_range_parameters( ) next_form.fill_from_trait(session_stored_simulator.coupling) rendering_rules = SimulatorFragmentRenderingRules( next_form, SimulatorWizzardURLs.SET_COUPLING_PARAMS_URL, SimulatorWizzardURLs.SET_CONNECTIVITY_URL, is_simulation_copy, is_simulation_load, self.context.last_loaded_fragment_url, cherrypy.request.method) return rendering_rules.to_dict() @expose_fragment('simulator_fragment') def set_coupling_params(self, **data): session_stored_simulator, is_simulation_copy, is_simulation_load, is_branch = self.context.get_common_params( ) if cherrypy.request.method == POST_REQUEST: self.context.add_last_loaded_form_url_to_session( SimulatorWizzardURLs.SET_SURFACE_URL) form = get_form_for_coupling( type(session_stored_simulator.coupling))() form.fill_from_post(data) form.fill_trait(session_stored_simulator.coupling) surface_fragment = self.algorithm_service.prepare_adapter_form( form_instance=SimulatorSurfaceFragment(), project_id=self.context.project.id) surface_fragment.fill_from_trait(session_stored_simulator.surface) rendering_rules = SimulatorFragmentRenderingRules( surface_fragment, SimulatorWizzardURLs.SET_SURFACE_URL, SimulatorWizzardURLs.SET_COUPLING_PARAMS_URL, is_simulation_copy, is_simulation_load, self.context.last_loaded_fragment_url, cherrypy.request.method, is_branch=is_branch) return rendering_rules.to_dict() @expose_fragment('simulator_fragment') def set_surface(self, **data): session_stored_simulator, is_simulation_copy, is_simulation_load, is_branch = self.context.get_common_params( ) rendering_rules = SimulatorFragmentRenderingRules( previous_form_action_url=SimulatorWizzardURLs.SET_SURFACE_URL, is_simulation_copy=is_simulation_copy, is_simulation_readonly_load=is_simulation_load, last_form_url=self.context.last_loaded_fragment_url, last_request_type=cherrypy.request.method, is_branch=is_branch) if cherrypy.request.method == POST_REQUEST: form = SimulatorSurfaceFragment() form.fill_from_post(data) self.simulator_service.reset_at_surface_change( is_simulation_copy, form, session_stored_simulator) form.fill_trait(session_stored_simulator) if session_stored_simulator.surface: self.context.add_last_loaded_form_url_to_session( SimulatorWizzardURLs.SET_CORTEX_URL) else: self.context.add_last_loaded_form_url_to_session( SimulatorWizzardURLs.SET_STIMULUS_URL) return SimulatorSurfaceFragment.prepare_next_fragment_after_surface( session_stored_simulator, rendering_rules, self.context.project.id, SimulatorWizzardURLs.SET_CORTEX_URL, SimulatorWizzardURLs.SET_STIMULUS_URL) @expose_fragment('simulator_fragment') def set_cortex(self, **data): session_stored_simulator, is_simulation_copy, is_simulation_load, _ = self.context.get_common_params( ) if cherrypy.request.method == POST_REQUEST: self.context.add_last_loaded_form_url_to_session( SimulatorWizzardURLs.SET_STIMULUS_URL) rm_fragment = SimulatorRMFragment() rm_fragment.fill_from_post(data) rm_fragment.fill_trait(session_stored_simulator.surface) rendering_rules = SimulatorFragmentRenderingRules( None, None, SimulatorWizzardURLs.SET_CORTEX_URL, is_simulation_copy, is_simulation_load, self.context.last_loaded_fragment_url, cherrypy.request.method) return SimulatorStimulusFragment.prepare_stimulus_fragment( session_stored_simulator, rendering_rules, True, SimulatorWizzardURLs.SET_STIMULUS_URL, self.context.project.id) @expose_fragment('simulator_fragment') def set_stimulus(self, **data): session_stored_simulator, is_simulation_copy, is_simulation_load, is_branch = self.context.get_common_params( ) if cherrypy.request.method == POST_REQUEST: self.context.add_last_loaded_form_url_to_session( SimulatorWizzardURLs.SET_MODEL_URL) stimuli_fragment = SimulatorStimulusFragment( session_stored_simulator.is_surface_simulation) stimuli_fragment.fill_from_post(data) stimuli_fragment.fill_trait(session_stored_simulator) model_fragment = self.algorithm_service.prepare_adapter_form( form_instance=SimulatorModelFragment()) model_fragment.fill_from_trait(session_stored_simulator) rendering_rules = SimulatorFragmentRenderingRules( model_fragment, SimulatorWizzardURLs.SET_MODEL_URL, SimulatorWizzardURLs.SET_STIMULUS_URL, is_simulation_copy, is_simulation_load, self.context.last_loaded_fragment_url, cherrypy.request.method, is_model_fragment=True, is_surface_simulation=session_stored_simulator. is_surface_simulation, is_branch=is_branch) return rendering_rules.to_dict() @expose_fragment('simulator_fragment') def set_model(self, **data): session_stored_simulator, is_simulation_copy, is_simulation_load, _ = self.context.get_common_params( ) if cherrypy.request.method == POST_REQUEST: self.context.add_last_loaded_form_url_to_session( SimulatorWizzardURLs.SET_MODEL_PARAMS_URL) form = SimulatorModelFragment() form.fill_from_post(data) form.fill_trait(session_stored_simulator) form = self.algorithm_service.prepare_adapter_form( form_instance=get_form_for_model( type(session_stored_simulator.model))()) self.range_parameters.model_parameters = form.get_range_parameters() form.fill_from_trait(session_stored_simulator.model) rendering_rules = SimulatorFragmentRenderingRules( form, SimulatorWizzardURLs.SET_MODEL_PARAMS_URL, SimulatorWizzardURLs.SET_MODEL_URL, is_simulation_copy, is_simulation_load, self.context.last_loaded_fragment_url, cherrypy.request.method) return rendering_rules.to_dict() @expose_fragment('simulator_fragment') def set_model_params(self, **data): session_stored_simulator, is_simulation_copy, is_simulation_load, is_branch = self.context.get_common_params( ) if cherrypy.request.method == POST_REQUEST: self.context.add_last_loaded_form_url_to_session( SimulatorWizzardURLs.SET_INTEGRATOR_URL) form = get_form_for_model(type(session_stored_simulator.model))() form.fill_from_post(data) form.fill_trait(session_stored_simulator.model) integrator_fragment = self.algorithm_service.prepare_adapter_form( form_instance=SimulatorIntegratorFragment()) integrator_fragment.integrator.display_subform = False integrator_fragment.fill_from_trait(session_stored_simulator) rendering_rules = SimulatorFragmentRenderingRules( integrator_fragment, SimulatorWizzardURLs.SET_INTEGRATOR_URL, SimulatorWizzardURLs.SET_MODEL_PARAMS_URL, is_simulation_copy, is_simulation_load, self.context.last_loaded_fragment_url, cherrypy.request.method, is_branch=is_branch) return rendering_rules.to_dict() @expose_fragment('simulator_fragment') def set_integrator(self, **data): session_stored_simulator, is_simulation_copy, is_simulation_load, _ = self.context.get_common_params( ) if cherrypy.request.method == POST_REQUEST: self.context.add_last_loaded_form_url_to_session( SimulatorWizzardURLs.SET_INTEGRATOR_PARAMS_URL) fragment = SimulatorIntegratorFragment() fragment.fill_from_post(data) fragment.fill_trait(session_stored_simulator) form = self.algorithm_service.prepare_adapter_form( form_instance=get_form_for_integrator( type(session_stored_simulator.integrator))()) if hasattr(form, 'noise'): form.noise.display_subform = False form.fill_from_trait(session_stored_simulator.integrator) rendering_rules = SimulatorFragmentRenderingRules( form, SimulatorWizzardURLs.SET_INTEGRATOR_PARAMS_URL, SimulatorWizzardURLs.SET_INTEGRATOR_URL, is_simulation_copy, is_simulation_load, self.context.last_loaded_fragment_url, cherrypy.request.method) return rendering_rules.to_dict() @expose_fragment('simulator_fragment') def set_integrator_params(self, **data): session_stored_simulator, is_simulation_copy, is_simulation_load, _ = self.context.get_common_params( ) if cherrypy.request.method == POST_REQUEST: form = get_form_for_integrator( type(session_stored_simulator.integrator))() form.fill_from_post(data) form.fill_trait(session_stored_simulator.integrator) if isinstance(session_stored_simulator.integrator, IntegratorStochasticViewModel): self.context.add_last_loaded_form_url_to_session( SimulatorWizzardURLs.SET_NOISE_PARAMS_URL) else: self.context.add_last_loaded_form_url_to_session( SimulatorWizzardURLs.SET_MONITORS_URL) rendering_rules = SimulatorFragmentRenderingRules( None, None, SimulatorWizzardURLs.SET_INTEGRATOR_PARAMS_URL, is_simulation_copy, is_simulation_load, self.context.last_loaded_fragment_url, cherrypy.request.method, is_noise_fragment=False) if not isinstance(session_stored_simulator.integrator, IntegratorStochasticViewModel): return self.monitors_handler.prepare_monitor_fragment( session_stored_simulator, rendering_rules, SimulatorWizzardURLs.SET_MONITORS_URL) integrator_noise_fragment = get_form_for_noise( type(session_stored_simulator.integrator.noise))() if hasattr(integrator_noise_fragment, 'equation'): integrator_noise_fragment.equation.display_subform = False self.range_parameters.integrator_noise_parameters = integrator_noise_fragment.get_range_parameters( ) integrator_noise_fragment.fill_from_trait( session_stored_simulator.integrator.noise) rendering_rules.form = integrator_noise_fragment rendering_rules.form_action_url = SimulatorWizzardURLs.SET_NOISE_PARAMS_URL rendering_rules.is_noise_fragment = True return rendering_rules.to_dict() @expose_fragment('simulator_fragment') def set_noise_params(self, **data): session_stored_simulator, is_simulation_copy, is_simulation_load, is_branch = self.context.get_common_params( ) if cherrypy.request.method == POST_REQUEST: form = get_form_for_noise( type(session_stored_simulator.integrator.noise))() form.fill_from_post(data) form.fill_trait(session_stored_simulator.integrator.noise) if isinstance(session_stored_simulator.integrator.noise, AdditiveNoiseViewModel): self.context.add_last_loaded_form_url_to_session( SimulatorWizzardURLs.SET_MONITORS_URL) else: self.context.add_last_loaded_form_url_to_session( SimulatorWizzardURLs.SET_NOISE_EQUATION_PARAMS_URL) rendering_rules = SimulatorFragmentRenderingRules( None, None, SimulatorWizzardURLs.SET_NOISE_PARAMS_URL, is_simulation_copy, is_simulation_load, self.context.last_loaded_fragment_url, cherrypy.request.method) return self.monitors_handler.prepare_next_fragment_after_noise( session_stored_simulator, is_branch, rendering_rules, SimulatorWizzardURLs.SET_MONITORS_URL, SimulatorWizzardURLs.SET_NOISE_EQUATION_PARAMS_URL) @expose_fragment('simulator_fragment') def set_noise_equation_params(self, **data): session_stored_simulator, is_simulation_copy, is_simulation_load, _ = self.context.get_common_params( ) if cherrypy.request.method == POST_REQUEST: self.context.add_last_loaded_form_url_to_session( SimulatorWizzardURLs.SET_MONITORS_URL) form = get_form_for_equation( type(session_stored_simulator.integrator.noise.b))() form.fill_from_post(data) form.fill_trait(session_stored_simulator.integrator.noise.b) rendering_rules = SimulatorFragmentRenderingRules( None, None, SimulatorWizzardURLs.SET_NOISE_EQUATION_PARAMS_URL, is_simulation_copy, is_simulation_load, self.context.last_loaded_fragment_url, cherrypy.request.method) return self.monitors_handler.prepare_monitor_fragment( session_stored_simulator, rendering_rules, SimulatorWizzardURLs.SET_MONITORS_URL) @staticmethod def build_monitor_url(fragment_url, monitor): url_regex = '{}/{}' url = url_regex.format(fragment_url, monitor) return url def get_first_monitor_fragment_url(self, simulator, monitors_url): first_monitor = simulator.first_monitor if first_monitor is not None: monitor_vm_name = type(first_monitor).__name__ return self.build_monitor_url(monitors_url, monitor_vm_name) return SimulatorWizzardURLs.SETUP_PSE_URL @expose_fragment('simulator_fragment') def set_monitors(self, **data): session_stored_simulator, is_simulation_copy, is_simulation_load, is_branch = self.context.get_common_params( ) if cherrypy.request.method == POST_REQUEST: fragment = SimulatorMonitorFragment( is_surface_simulation=session_stored_simulator. is_surface_simulation) fragment.fill_from_post(data) self.monitors_handler.set_monitors_list_on_simulator( session_stored_simulator, fragment.monitors.value) last_loaded_fragment_url = self.get_first_monitor_fragment_url( session_stored_simulator, SimulatorWizzardURLs.SET_MONITOR_PARAMS_URL) if cherrypy.request.method == POST_REQUEST: self.context.add_last_loaded_form_url_to_session( last_loaded_fragment_url) rendering_rules = SimulatorFragmentRenderingRules( is_simulation_copy=is_simulation_copy, is_simulation_readonly_load=is_simulation_load, last_request_type=cherrypy.request.method, form_action_url=last_loaded_fragment_url, previous_form_action_url=SimulatorWizzardURLs.SET_MONITORS_URL) return self.monitors_handler.get_fragment_after_monitors( session_stored_simulator, self.context.burst_config, self.context.project.id, is_branch, rendering_rules, SimulatorWizzardURLs.SETUP_PSE_URL) def get_url_after_monitors(self, current_monitor, monitor_name, next_monitor): if isinstance(current_monitor, BoldViewModel): return self.build_monitor_url( SimulatorWizzardURLs.SET_MONITOR_EQUATION_URL, monitor_name) if next_monitor is not None: return self.build_monitor_url( SimulatorWizzardURLs.SET_MONITOR_PARAMS_URL, type(next_monitor).__name__) return SimulatorWizzardURLs.SETUP_PSE_URL @staticmethod def get_url_for_final_fragment(burst_config): if burst_config.is_pse_burst(): return SimulatorWizzardURLs.LAUNCH_PSE_URL return SimulatorWizzardURLs.SETUP_PSE_URL def get_urls_for_next_monitor_fragment(self, next_monitor, current_monitor): form_action_url = self.build_monitor_url( SimulatorWizzardURLs.SET_MONITOR_PARAMS_URL, type(next_monitor).__name__) if_bold_url = self.build_monitor_url( SimulatorWizzardURLs.SET_MONITOR_EQUATION_URL, type(current_monitor).__name__) return form_action_url, if_bold_url @expose_fragment('simulator_fragment') def set_monitor_params(self, current_monitor_name, **data): session_stored_simulator, is_simulation_copy, is_simulator_load, is_branch = self.context.get_common_params( ) current_monitor, next_monitor = self.monitors_handler.get_current_and_next_monitor_form( current_monitor_name, session_stored_simulator) if cherrypy.request.method == POST_REQUEST: form = get_form_for_monitor( type(current_monitor))(session_stored_simulator) form.fill_from_post(data) form.fill_trait(current_monitor) last_loaded_form_url = self.get_url_after_monitors( current_monitor, current_monitor_name, next_monitor) self.context.add_last_loaded_form_url_to_session( last_loaded_form_url) previous_form_action_url = self.build_monitor_url( SimulatorWizzardURLs.SET_MONITOR_PARAMS_URL, current_monitor_name) rendering_rules = SimulatorFragmentRenderingRules( is_simulation_copy=is_simulation_copy, is_simulation_readonly_load=is_simulator_load, last_request_type=cherrypy.request.method, last_form_url=self.context.last_loaded_fragment_url, previous_form_action_url=previous_form_action_url) form_action_url, if_bold_url = self.get_urls_for_next_monitor_fragment( next_monitor, current_monitor) return self.monitors_handler.handle_next_fragment_for_monitors( self.context, rendering_rules, current_monitor, next_monitor, False, form_action_url, if_bold_url) def get_url_after_monitor_equation(self, next_monitor): if next_monitor is None: return SimulatorWizzardURLs.SETUP_PSE_URL last_loaded_fragment_url = self.build_monitor_url( SimulatorWizzardURLs.SET_MONITOR_PARAMS_URL, type(next_monitor).__name__) return last_loaded_fragment_url @expose_fragment('simulator_fragment') def set_monitor_equation(self, current_monitor_name, **data): session_stored_simulator, is_simulation_copy, is_simulator_load, is_branch = self.context.get_common_params( ) current_monitor, next_monitor = self.monitors_handler.get_current_and_next_monitor_form( current_monitor_name, session_stored_simulator) if cherrypy.request.method == POST_REQUEST: form = get_form_for_equation(type(current_monitor.hrf_kernel))() form.fill_from_post(data) form.fill_trait(current_monitor.hrf_kernel) last_loaded_fragment_url = self.get_url_after_monitor_equation( next_monitor) self.context.add_last_loaded_form_url_to_session( last_loaded_fragment_url) previous_form_action_url = self.build_monitor_url( SimulatorWizzardURLs.SET_MONITOR_EQUATION_URL, current_monitor_name) rendering_rules = SimulatorFragmentRenderingRules( None, None, previous_form_action_url, is_simulation_copy, is_simulator_load, self.context.last_loaded_fragment_url, cherrypy.request) form_action_url, if_bold_url = self.get_urls_for_next_monitor_fragment( next_monitor, current_monitor) return self.monitors_handler.handle_next_fragment_for_monitors( self.context, rendering_rules, current_monitor, next_monitor, True, form_action_url, if_bold_url) @expose_fragment('simulator_fragment') def setup_pse(self, **data): session_stored_simulator, is_simulation_copy, is_simulator_load, _ = self.context.get_common_params( ) burst_config = self.context.burst_config all_range_parameters = self.range_parameters.get_all_range_parameters() next_form = self.algorithm_service.prepare_adapter_form( form_instance=SimulatorPSEConfigurationFragment( self.range_parameters.get_all_range_parameters())) if cherrypy.request.method == POST_REQUEST: session_stored_simulator.simulation_length = float( data['simulation_length']) burst_config.name = data['input_simulation_name_id'] self.context.add_last_loaded_form_url_to_session( SimulatorWizzardURLs.SET_PSE_PARAMS_URL) param1, param2 = self.burst_service.handle_range_params_at_loading( burst_config, all_range_parameters) if param1: param_dict = {'pse_param1': param1.name} if param2 is not None: param_dict['pse_param2'] = param2.name next_form.fill_from_post(param_dict) rendering_rules = SimulatorFragmentRenderingRules( next_form, SimulatorWizzardURLs.SET_PSE_PARAMS_URL, SimulatorWizzardURLs.SETUP_PSE_URL, is_simulation_copy, is_simulator_load, self.context.last_loaded_fragment_url, cherrypy.request.method) return rendering_rules.to_dict() @expose_fragment('simulator_fragment') def set_pse_params(self, **data): session_stored_simulator, is_simulation_copy, is_simulation_load, _ = self.context.get_common_params( ) burst_config = self.context.burst_config form = SimulatorPSEConfigurationFragment( self.range_parameters.get_all_range_parameters()) if cherrypy.request.method == POST_REQUEST: self.context.add_last_loaded_form_url_to_session( SimulatorWizzardURLs.LAUNCH_PSE_URL) form.fill_from_post(data) param1 = form.pse_param1.value burst_config.range1 = param1.to_json() param2 = None if form.pse_param2.value: param2 = form.pse_param2.value burst_config.range2 = param2.to_json() else: all_range_parameters = self.range_parameters.get_all_range_parameters( ) param1, param2 = self.burst_service.handle_range_params_at_loading( burst_config, all_range_parameters) next_form = self.algorithm_service.prepare_adapter_form( form_instance=SimulatorPSERangeFragment(param1, param2)) rendering_rules = SimulatorFragmentRenderingRules( next_form, SimulatorWizzardURLs.LAUNCH_PSE_URL, SimulatorWizzardURLs.SET_PSE_PARAMS_URL, is_simulation_copy, is_simulation_load, last_form_url=self.context.last_loaded_fragment_url, is_launch_pse_fragment=True) return rendering_rules.to_dict() @expose_json def launch_pse(self, **data): session_stored_simulator = self.context.simulator all_range_parameters = self.range_parameters.get_all_range_parameters() range_param1, range_param2 = SimulatorPSERangeFragment.fill_from_post( all_range_parameters, **data) burst_config = self.context.burst_config burst_config.start_time = datetime.now() burst_config.range1 = range_param1.to_json() if range_param2: burst_config.range2 = range_param2.to_json() burst_config = self.burst_service.prepare_burst_for_pse(burst_config) session_stored_simulator.operation_group_gid = uuid.UUID( burst_config.operation_group.gid) session_stored_simulator.ranges = json.dumps(burst_config.ranges) try: thread = threading.Thread( target=self.simulator_service.async_launch_and_prepare_pse, kwargs={ 'burst_config': burst_config, 'user': self.context.logged_user, 'project': self.context.project, 'simulator_algo': self.cached_simulator_algorithm, 'range_param1': range_param1, 'range_param2': range_param2, 'session_stored_simulator': session_stored_simulator }) thread.start() return {'id': burst_config.id} except BurstServiceException as e: self.logger.exception("Could not launch burst!") return {'error': e.message} @expose_json def launch_simulation(self, launch_mode, **data): current_form = SimulatorFinalFragment() burst_config = self.context.burst_config burst_config.range1 = None burst_config.range2 = None try: current_form.fill_from_post(data) except Exception as exc: self.logger.exception(exc) return {'error': str(exc)} burst_name = current_form.simulation_name.value session_stored_simulator = self.context.simulator session_stored_simulator.simulation_length = current_form.simulation_length.value if burst_name != 'none_undefined': burst_config.name = burst_name if launch_mode == self.burst_service.LAUNCH_BRANCH: simulation_state_index = self.simulator_service.get_simulation_state_index( burst_config, SimulationHistoryIndex) session_stored_simulator.history_gid = simulation_state_index[ 0].gid burst_config.start_time = datetime.now() session_burst_config = self.burst_service.store_burst(burst_config) try: thread = threading.Thread(target=self.simulator_service. async_launch_and_prepare_simulation, kwargs={ 'burst_config': session_burst_config, 'user': self.context.logged_user, 'project': self.context.project, 'simulator_algo': self.cached_simulator_algorithm, 'simulator': session_stored_simulator }) thread.start() return {'id': session_burst_config.id} except BurstServiceException as e: self.logger.exception('Could not launch burst!') return {'error': e.message} @expose_fragment('burst/burst_history') def load_burst_history(self): """ Load the available burst that are stored in the database at this time. This is one alternative to 'chrome-back problem'. """ bursts = self.burst_service.get_available_bursts( self.context.project.id) self.burst_service.populate_burst_disk_usage(bursts) return { 'burst_list': bursts, 'selectedBurst': self.context.burst_config.id, 'first_fragment_url': SimulatorFragmentRenderingRules.FIRST_FORM_URL } @cherrypy.expose def get_last_fragment_url(self, burst_config_id): burst_config = self.burst_service.load_burst_configuration( burst_config_id) return self.get_url_for_final_fragment(burst_config) @expose_fragment('simulator_fragment') def load_burst_read_only(self, burst_config_id): try: burst_config = self.burst_service.load_burst_configuration( burst_config_id) storage_path = self.files_helper.get_project_folder( self.context.project, str(burst_config.fk_simulation)) simulator = h5.load_view_model(burst_config.simulator_gid, storage_path) last_loaded_form_url = self.get_url_for_final_fragment( burst_config) self.context.init_session_at_burst_loading(burst_config, simulator, last_loaded_form_url) form = self.prepare_first_fragment() self.monitors_handler.build_list_of_monitors_from_view_models( self.context.simulator) rendering_rules = SimulatorFragmentRenderingRules( form, SimulatorWizzardURLs.SET_CONNECTIVITY_URL, is_simulation_readonly_load=True, is_first_fragment=True) return rendering_rules.to_dict() except Exception: # Most probably Burst was removed. Delete it from session, so that client # has a good chance to get a good response on refresh self.logger.exception("Error loading burst") self.context.remove_burst_config_from_session() raise def _prepare_first_fragment_for_burst_copy(self, burst_config_id, burst_name_format): simulator, burst_config_copy = self.burst_service.prepare_data_for_burst_copy( burst_config_id, burst_name_format, self.context.project) self.monitors_handler.build_list_of_monitors_from_view_models( simulator) last_loaded_form_url = self.get_url_for_final_fragment( burst_config_copy) self.context.init_session_at_copy_preparation(burst_config_copy, simulator, last_loaded_form_url) return self.prepare_first_fragment() @expose_fragment('simulator_fragment') def copy_simulator_configuration(self, burst_config_id): self.context.add_branch_and_copy_to_session(False, True) form = self._prepare_first_fragment_for_burst_copy( burst_config_id, self.COPY_NAME_FORMAT) rendering_rules = SimulatorFragmentRenderingRules( form, SimulatorWizzardURLs.SET_CONNECTIVITY_URL, is_simulation_copy=True, is_simulation_readonly_load=True, is_first_fragment=True) return rendering_rules.to_dict() @expose_fragment('simulator_fragment') def branch_simulator_configuration(self, burst_config_id): self.context.add_branch_and_copy_to_session(True, False) form = self._prepare_first_fragment_for_burst_copy( burst_config_id, self.BRANCH_NAME_FORMAT) rendering_rules = SimulatorFragmentRenderingRules( form, SimulatorWizzardURLs.SET_CONNECTIVITY_URL, is_simulation_copy=True, is_simulation_readonly_load=True, is_first_fragment=True) return rendering_rules.to_dict() @expose_fragment('simulator_fragment') def reset_simulator_configuration(self): burst_config = BurstConfiguration(self.context.project.id) self.context.init_session_at_sim_reset( burst_config, SimulatorWizzardURLs.SET_CONNECTIVITY_URL) self.monitors_handler.clear_next_monitors_dict() form = self.prepare_first_fragment() rendering_rules = SimulatorFragmentRenderingRules( form, SimulatorWizzardURLs.SET_CONNECTIVITY_URL, is_first_fragment=True) return rendering_rules.to_dict() @expose_json def rename_burst(self, burst_id, burst_name): """ Rename the burst given by burst_id, setting it's new name to burst_name. """ validation_result = SimulatorFinalFragment.is_burst_name_ok(burst_name) if validation_result is True: self.burst_service.rename_burst(burst_id, burst_name) return {'success': "Simulation successfully renamed!"} else: self.logger.exception(validation_result) return {'error': validation_result} @expose_json def get_history_status(self, **data): """ For each burst id received, get the status and return it. """ return self.burst_service.update_history_status( json.loads(data['burst_ids'])) @cherrypy.expose @handle_error(redirect=False) @check_user def export(self, burst_id): export_manager = ExportManager() export_zip = export_manager.export_simulator_configuration(burst_id) result_name = "tvb_simulation_" + str(burst_id) + ".zip" return serve_file(export_zip, "application/x-download", "attachment", result_name) @expose_fragment("overlay") def get_upload_overlay(self): template_specification = self.fill_overlay_attributes( None, "Upload", "Simulation ZIP", "burst/upload_burst_overlay", "dialog-upload") template_specification[ 'first_fragment_url'] = SimulatorWizzardURLs.SET_CONNECTIVITY_URL return self.fill_default_attributes(template_specification) @cherrypy.expose @handle_error(redirect=True) @check_user @settings def load_simulator_configuration_from_zip(self, **data): """Upload Simulator from previously exported ZIP file""" self.logger.debug("Uploading ..." + str(data)) last_loaded_form_url = SimulatorWizzardURLs.SETUP_PSE_URL try: upload_param = "uploadedfile" if upload_param in data and data[upload_param]: simulator, burst_config, sim_folder = self.burst_service.load_simulation_from_zip( data[upload_param], self.context.project) dts_folder = os.path.join( sim_folder, ExportManager.EXPORTED_SIMULATION_DTS_DIR) ImportService().import_project_operations( self.context.project, dts_folder, False, None) self.monitors_handler.build_list_of_monitors_from_view_models( simulator) if burst_config.is_pse_burst(): last_loaded_form_url = SimulatorWizzardURLs.LAUNCH_PSE_URL self.context.init_session_at_sim_config_from_zip( burst_config, simulator, last_loaded_form_url) except IOError as ioexcep: self.logger.exception(ioexcep) self.context.set_warning_message( "This ZIP does not contain a complete simulator configuration") except ServicesBaseException as excep: self.logger.warning(excep.message) self.context.set_warning_message(excep.message) raise cherrypy.HTTPRedirect('/burst/')
def launch(self, view_model): # type: (ZIPConnectivityImporterModel) -> [ConnectivityIndex] """ Execute import operations: unpack ZIP and build Connectivity object as result. :raises LaunchException: when `uploaded` is empty or nonexistent :raises Exception: when * weights or tracts matrix is invalid (negative values, wrong shape) * any of the vector orientation, areas, cortical or hemisphere is \ different from the expected number of nodes """ if view_model.uploaded is None: raise LaunchException( "Please select ZIP file which contains data to import") files = FilesHelper().unpack_zip(view_model.uploaded, self.storage_path) weights_matrix = None centres = None labels_vector = None tract_matrix = None orientation = None areas = None cortical_vector = None hemisphere_vector = None for file_name in files: file_name_low = file_name.lower() if self.WEIGHT_TOKEN in file_name_low: weights_matrix = self.read_list_data(file_name) elif self.CENTRES_TOKEN in file_name_low or self.CENTRES_TOKEN2 in file_name_low: centres = self.read_list_data(file_name, usecols=[1, 2, 3]) labels_vector = self.read_list_data(file_name, dtype=numpy.str, usecols=[0]) elif self.TRACT_TOKEN in file_name_low: tract_matrix = self.read_list_data(file_name) elif self.ORIENTATION_TOKEN in file_name_low: orientation = self.read_list_data(file_name) elif self.AREA_TOKEN in file_name_low: areas = self.read_list_data(file_name) elif self.CORTICAL_INFO in file_name_low: cortical_vector = self.read_list_data(file_name, dtype=numpy.bool) elif self.HEMISPHERE_INFO in file_name_low: hemisphere_vector = self.read_list_data(file_name, dtype=numpy.bool) # Clean remaining text-files. FilesHelper.remove_files(files, True) result = Connectivity() # Fill positions if centres is None: raise Exception( "Region centres are required for Connectivity Regions! " "We expect a file that contains *centres* inside the uploaded ZIP." ) expected_number_of_nodes = len(centres) if expected_number_of_nodes < 2: raise Exception("A connectivity with at least 2 nodes is expected") result.centres = centres if labels_vector is not None: result.region_labels = labels_vector # Fill and check weights if weights_matrix is not None: if weights_matrix.shape != (expected_number_of_nodes, expected_number_of_nodes): raise Exception( "Unexpected shape for weights matrix! " "Should be %d x %d " % (expected_number_of_nodes, expected_number_of_nodes)) result.weights = weights_matrix if view_model.normalization: result.weights = result.scaled_weights( view_model.normalization) # Fill and check tracts. Allow empty files for tracts, they will be computed by tvb-library. if tract_matrix is not None: if tract_matrix.size != 0: if numpy.any([x < 0 for x in tract_matrix.flatten()]): raise Exception( "Negative values are not accepted in tracts matrix! " "Please check your file, and use values >= 0") if tract_matrix.shape != (expected_number_of_nodes, expected_number_of_nodes): raise Exception( "Unexpected shape for tracts matrix! " "Should be %d x %d " % (expected_number_of_nodes, expected_number_of_nodes)) result.tract_lengths = tract_matrix if orientation is not None: if len(orientation) != expected_number_of_nodes: raise Exception( "Invalid size for vector orientation. " "Expected the same as region-centers number %d" % expected_number_of_nodes) result.orientations = orientation if areas is not None: if len(areas) != expected_number_of_nodes: raise Exception( "Invalid size for vector areas. " "Expected the same as region-centers number %d" % expected_number_of_nodes) result.areas = areas if cortical_vector is not None: if len(cortical_vector) != expected_number_of_nodes: raise Exception( "Invalid size for vector cortical. " "Expected the same as region-centers number %d" % expected_number_of_nodes) result.cortical = cortical_vector if hemisphere_vector is not None: if len(hemisphere_vector) != expected_number_of_nodes: raise Exception( "Invalid size for vector hemispheres. " "Expected the same as region-centers number %d" % expected_number_of_nodes) result.hemispheres = hemisphere_vector result.configure() return h5.store_complete(result, self.storage_path)
def __init__(self): self.logger = get_logger(self.__class__.__module__) self.file_helper = FilesHelper() self.input_tree_manager = InputTreeManager()
def __init__(self): self.logger = get_logger(__name__) self.user_id = None self.files_helper = FilesHelper() self.created_projects = []
class TVBLoader(object): def __init__(self, registry): self.file_handler = FilesHelper() self.registry = registry def path_for_stored_index(self, dt_index_instance): # type: (DataType) -> str """ Given a Datatype(HasTraitsIndex) instance, build where the corresponding H5 should be or is stored""" operation = dao.get_operation_by_id( dt_index_instance.fk_from_operation) operation_folder = self.file_handler.get_project_folder( operation.project, str(operation.id)) gid = uuid.UUID(dt_index_instance.gid) h5_file_class = self.registry.get_h5file_for_index( dt_index_instance.__class__) fname = get_h5_filename(h5_file_class.file_name_base(), gid) return os.path.join(operation_folder, fname) def path_for(self, operation_dir, h5_file_class, gid, dt_class=None): if isinstance(gid, str): gid = uuid.UUID(gid) fname = get_h5_filename(dt_class or h5_file_class.file_name_base(), gid) return os.path.join(operation_dir, fname) def load_from_index(self, dt_index): # type: (DataType) -> HasTraits h5_path = self.path_for_stored_index(dt_index) h5_file_class = self.registry.get_h5file_for_index(dt_index.__class__) traits_class = self.registry.get_datatype_for_index(dt_index) with h5_file_class(h5_path) as f: result_dt = traits_class() f.load_into(result_dt) return result_dt def load_complete_by_function(self, file_path, load_ht_function): # type: (str, callable) -> (HasTraits, GenericAttributes) with H5File.from_file(file_path) as f: try: datatype_cls = self.registry.get_datatype_for_h5file(f) except KeyError: datatype_cls = f.determine_datatype_from_file() datatype = datatype_cls() f.load_into(datatype) ga = f.load_generic_attributes() sub_dt_refs = f.gather_references(datatype_cls) for traited_attr, sub_gid in sub_dt_refs: if sub_gid is None: continue is_monitor = False if isinstance(sub_gid, list): sub_gid = sub_gid[0] is_monitor = True ref_ht = load_ht_function(sub_gid, traited_attr) if is_monitor: ref_ht = [ref_ht] setattr(datatype, traited_attr.field_name, ref_ht) return datatype, ga def load_with_references(self, file_path): def load_ht_function(sub_gid, traited_attr): ref_idx = dao.get_datatype_by_gid(sub_gid.hex, load_lazy=False) ref_ht = self.load_from_index(ref_idx) return ref_ht return self.load_complete_by_function(file_path, load_ht_function) def load_with_links(self, file_path): def load_ht_function(sub_gid, traited_attr): # Used traited_attr.default for cases similar to ProjectionMonitor which has obsnoise of type Noise and # it cannot be instantiated due to abstract methods, while the default is Additive() ref_ht = traited_attr.default or traited_attr.field_type() ref_ht.gid = sub_gid return ref_ht return self.load_complete_by_function(file_path, load_ht_function)
class DatatypesFactory(): """ This class provides a set of methods that helps user to create different data types for testing. These data types will be automatically stored in DB and file system if needed. """ USER_FULL_NAME = "Datatype Factory User" DATATYPE_STATE = "RAW_DATA" DATATYPE_DATA = ["test", "for", "datatypes", "factory"] DATATYPE_MEASURE_METRIC = {'v': 3} RANGE_1 = ["row1", [1, 2, 3]] RANGE_2 = ["row2", [0.1, 0.3, 0.5]] user = None project = None operation = None def __init__(self): micro_postfix = "_%d" % int(time.time() * 1000000) # Here create all structures needed later for data types creation self.files_helper = FilesHelper() # First create user user = model.User("datatype_factory_user" + micro_postfix, "test_pass", "*****@*****.**" + micro_postfix, True, "user") self.user = dao.store_entity(user) # Now create a project project_service = ProjectService() data = dict(name='DatatypesFactoryProject' + micro_postfix, description='test_desc', users=[]) self.project = project_service.store_project(self.user, True, None, **data) # Create algorithm alg_category = model.AlgorithmCategory('one', True) dao.store_entity(alg_category) alg_group = model.AlgorithmGroup("test_module1", "classname1", alg_category.id) dao.store_entity(alg_group) algorithm = model.Algorithm(alg_group.id, 'id', name='', req_data='', param_name='', output='') self.algorithm = dao.store_entity(algorithm) #Create an operation self.meta = {DataTypeMetaData.KEY_SUBJECT: self.USER_FULL_NAME, DataTypeMetaData.KEY_STATE: self.DATATYPE_STATE} operation = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters', meta=json.dumps(self.meta), status=model.STATUS_FINISHED, method_name=ABCAdapter.LAUNCH_METHOD) self.operation = dao.store_entity(operation) def get_project(self): """ Return project to which generated data types are assigned """ return self.project def get_operation(self): """ Return operation to which generated data types are assigned """ return self.operation def get_user(self): """ Return user to which generated data types are assigned """ return self.user def _store_datatype(self, data_type, operation_id=None): """ Launch adapter to store a create a persistent DataType. """ operation_id = operation_id or self.operation.id data_type.type = data_type.__class__.__name__ data_type.module = data_type.__class__.__module__ data_type.subject = self.USER_FULL_NAME data_type.state = self.DATATYPE_STATE data_type.set_operation_id(operation_id) adapter_instance = StoreAdapter([data_type]) operation = dao.get_operation_by_id(operation_id) OperationService().initiate_prelaunch(operation, adapter_instance, {}) return data_type def create_simple_datatype(self, subject=USER_FULL_NAME, state=DATATYPE_STATE): """ This method creates a simple data type """ datatype_inst = Datatype1() self._fill_datatype(datatype_inst, subject, state) # Store data type return self._store_datatype(datatype_inst) def create_datatype_with_storage(self, subject=USER_FULL_NAME, state=DATATYPE_STATE, data=DATATYPE_DATA, operation_id=None): """ This method creates and stores a data type which imply storage on the file system. """ datatype_inst = Datatype2() self._fill_datatype(datatype_inst, subject, state, operation_id) datatype_inst.string_data = data return self._store_datatype(datatype_inst, operation_id) def _fill_datatype(self, datatype, subject, state, operation_id=None): """ This method sets some common attributes on dataType """ operation_id = operation_id or self.operation.id datatype.subject = subject datatype.state = state # Set_operation_id also sets storage_path attribute datatype.set_operation_id(operation_id) def __create_operation(self): """ Create a operation entity. Return the operation, algo_id and the storage path. """ meta = {DataTypeMetaData.KEY_SUBJECT: "John Doe", DataTypeMetaData.KEY_STATE: "RAW_DATA"} algorithm, algo_group = FlowService().get_algorithm_by_module_and_class(SIMULATOR_MODULE, SIMULATOR_CLASS) operation = model.Operation(self.user.id, self.project.id, algo_group.id, json.dumps(''), meta=json.dumps(meta), status=model.STATUS_STARTED, method_name=ABCAdapter.LAUNCH_METHOD) operation = dao.store_entity(operation) storage_path = FilesHelper().get_project_folder(self.project, str(operation.id)) return operation, algorithm.id, storage_path def create_connectivity(self): """ Create a connectivity that will be used in "non-dummy" burst launches (with the actual simulator). """ operation, algo_id, storage_path = self.__create_operation() connectivity = Connectivity(storage_path=storage_path) connectivity.weights = numpy.ones((74, 74)) connectivity.centres = numpy.ones((74, 3)) adapter_instance = StoreAdapter([connectivity]) OperationService().initiate_prelaunch(operation, adapter_instance, {}) return algo_id, connectivity def create_timeseries(self, connectivity, ts_type=None, sensors=None): """ Create a stored TimeSeries entity. """ operation, _, storage_path = self.__create_operation() if ts_type == "EEG": time_series = TimeSeriesEEG(storage_path=storage_path, sensors=sensors) else: rm = dao.get_generic_entity(RegionMapping, connectivity.gid, '_connectivity') if len(rm) < 1: rm = None else: rm = rm[0] time_series = TimeSeriesRegion(storage_path=storage_path, connectivity=connectivity, region_mapping=rm) data = numpy.random.random((10, 10, 10, 10)) time_series.write_data_slice(data) time_series.write_time_slice(numpy.arange(10)) adapter_instance = StoreAdapter([time_series]) OperationService().initiate_prelaunch(operation, adapter_instance, {}) time_series = dao.get_datatype_by_gid(time_series.gid) return time_series def create_covariance(self, time_series): """ :returns: a stored DataType Covariance. """ operation, _, storage_path = self.__create_operation() covariance = Covariance(storage_path=storage_path, source=time_series) covariance.write_data_slice(numpy.random.random((10, 10, 10))) adapter_instance = StoreAdapter([covariance]) OperationService().initiate_prelaunch(operation, adapter_instance, {}) return covariance def create_crosscoherence(self, time_series): """ :returns: a stored entity of type CoherenceSpectrum """ operation, _, storage_path = self.__create_operation() partial_coh = CoherenceSpectrum(array_data=numpy.random.random((10, 10, 10, 10)), use_storage=False) coherence = CoherenceSpectrum(source=time_series, storage_path=storage_path, frequency=0.1, nfft=256) coherence.write_data_slice(partial_coh) coherence.close_file() adapter_instance = StoreAdapter([coherence]) OperationService().initiate_prelaunch(operation, adapter_instance, {}) return coherence def create_crosscorrelation(self, time_series): """ :returns: `CrossCorrelation` stored entity. """ operation, _, storage_path = self.__create_operation() partial_corr = CrossCorrelation(array_data=numpy.random.random((10, 10, 10, 10, 10)), use_storage=False) crossc = CrossCorrelation(source=time_series, storage_path=storage_path, time=range(10)) crossc.write_data_slice(partial_corr) crossc.close_file() adapter_instance = StoreAdapter([crossc]) OperationService().initiate_prelaunch(operation, adapter_instance, {}) return crossc def create_surface(self): """ Create a dummy surface entity. :returns: (Algorithm Identifier, stored Surface entity) """ operation, algo_id, storage_path = self.__create_operation() surface = CorticalSurface(storage_path=storage_path) surface.vertices = numpy.array([[-10, 0, 0], [0, 0, -10], [10, 0, 0], [0, 10, 0]], dtype=float) surface.triangles = numpy.array([[0, 1, 2], [0, 1, 3], [1, 2, 3], [0, 2, 3]], dtype=int) surface.number_of_triangles = 4 surface.number_of_vertices = 4 surface.triangle_normals = numpy.ones((4, 3)) surface.vertex_normals = numpy.ones((4, 3)) surface.zero_based_triangles = True surface.validate() adapter_instance = StoreAdapter([surface]) OperationService().initiate_prelaunch(operation, adapter_instance, {}) return algo_id, surface def create_connectivity_measure(self, connectivity): """ :returns: persisted entity ConnectivityMeasure """ operation, _, storage_path = self.__create_operation() conn_measure = ConnectivityMeasure(storage_path=storage_path) conn_measure.connectivity = connectivity adapter_instance = StoreAdapter([conn_measure]) OperationService().initiate_prelaunch(operation, adapter_instance, {}) return conn_measure def create_datatype_measure(self, analyzed_entity, operation=None, storage_path=None): """ :return: persisted DatatypeMeasure """ if operation is None: operation, _, storage_path = self.__create_operation() measure = DatatypeMeasure(storage_path=storage_path, metrics=self.DATATYPE_MEASURE_METRIC) measure.analyzed_datatype = analyzed_entity adapter_instance = StoreAdapter([measure]) OperationService().initiate_prelaunch(operation, adapter_instance, {}) return measure def create_ICA(self, timeseries): """ :returns: persisted entity IndependentComponents """ operation, _, storage_path = self.__create_operation() partial_ts = TimeSeries(use_storage=False) partial_ts.data = numpy.random.random((10, 10, 10, 10)) partial_ica = IndependentComponents(source=partial_ts, component_time_series=numpy.random.random((10, 10, 10, 10)), prewhitening_matrix=numpy.random.random((10, 10, 10, 10)), unmixing_matrix=numpy.random.random((10, 10, 10, 10)), n_components=10, use_storage=False) ica = IndependentComponents(source=timeseries, n_components=10, storage_path=storage_path) ica.write_data_slice(partial_ica) adapter_instance = StoreAdapter([ica]) OperationService().initiate_prelaunch(operation, adapter_instance, {}) return ica def create_datatype_group(self, subject=USER_FULL_NAME, state=DATATYPE_STATE, ): """ This method creates, stores and returns a DataTypeGroup entity. """ group = model.OperationGroup(self.project.id, ranges=[json.dumps(self.RANGE_1), json.dumps(self.RANGE_2)]) group = dao.store_entity(group) group_ms = model.OperationGroup(self.project.id, ranges=[json.dumps(self.RANGE_1), json.dumps(self.RANGE_2)]) group_ms = dao.store_entity(group_ms) datatype_group = model.DataTypeGroup(group, subject=subject, state=state, operation_id=self.operation.id) # Set storage path, before setting data datatype_group.storage_path = self.files_helper.get_project_folder(self.project, str(self.operation.id)) datatype_group = dao.store_entity(datatype_group) dt_group_ms = model.DataTypeGroup(group_ms, subject=subject, state=state, operation_id=self.operation.id) # Set storage path, before setting data dt_group_ms.storage_path = self.files_helper.get_project_folder(self.project, str(self.operation.id)) dao.store_entity(dt_group_ms) # Now create some data types and add them to group for range_val1 in self.RANGE_1[1]: for range_val2 in self.RANGE_2[1]: operation = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters', meta=json.dumps(self.meta), status=model.STATUS_FINISHED, method_name=ABCAdapter.LAUNCH_METHOD, range_values=json.dumps({self.RANGE_1[0]: range_val1, self.RANGE_2[0]: range_val2})) operation.fk_operation_group = group.id operation = dao.store_entity(operation) datatype = self.create_datatype_with_storage(operation_id=operation.id) datatype.number1 = range_val1 datatype.number2 = range_val2 datatype.fk_datatype_group = datatype_group.id datatype.set_operation_id(operation.id) dao.store_entity(datatype) op_ms = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters', meta=json.dumps(self.meta), status=model.STATUS_FINISHED, method_name=ABCAdapter.LAUNCH_METHOD, range_values=json.dumps({self.RANGE_1[0]: range_val1, self.RANGE_2[0]: range_val2})) op_ms.fk_operation_group = group_ms.id op_ms = dao.store_entity(op_ms) self.create_datatype_measure(datatype, op_ms, FilesHelper().get_project_folder(self.project, str(op_ms.id))) return datatype_group
def __init__(self, registry): self.file_handler = FilesHelper() self.registry = registry
def _store_view_model(operation, project, view_model): storage_path = FilesHelper().get_project_folder(project, str(operation.id)) h5.store_view_model(view_model, storage_path)
class FigureService: """ Service layer for Figure entities. """ _TYPE_PNG = "png" _TYPE_SVG = "svg" _BRANDING_BAR_PNG = os.path.join(os.path.dirname(__file__), "resources", "branding_bar.png") _BRANDING_BAR_SVG = os.path.join(os.path.dirname(__file__), "resources", "branding_bar.svg") _DEFAULT_SESSION_NAME = "Default" _DEFAULT_IMAGE_FILE_NAME = "snapshot." def __init__(self): self.logger = get_logger(self.__class__.__module__) self.file_helper = FilesHelper() def _write_png(self, store_path, export_data): img_data = base64.b64decode(export_data) # decode the image final_image = Image.open( StringIO(img_data)) # place it in a PIL stream branding_bar = Image.open( FigureService._BRANDING_BAR_PNG) # place the branding bar over final_image.paste(branding_bar, (0, final_image.size[1] - branding_bar.size[1]), branding_bar) final_image.save(store_path) # store to disk as PNG def _write_svg(self, store_path, export_data): dom = xml.dom.minidom.parseString(export_data) figureSvg = dom.getElementsByTagName('svg')[ 0] # get the original image dom = xml.dom.minidom.parse(FigureService._BRANDING_BAR_SVG) try: width = float(figureSvg.getAttribute('width').replace('px', '')) height = float(figureSvg.getAttribute('height').replace('px', '')) except ValueError: # defaults when dimensions are not given width = 1024 height = 768 figureSvg.setAttribute("width", str(width)) figureSvg.setAttribute("height", str(height)) finalSvg = dom.createElement('svg') # prepare the final svg brandingSvg = dom.getElementsByTagName('svg')[ 0] # get the branding bar brandingSvg.setAttribute("y", str(height)) # position it below the figure height += float(brandingSvg.getAttribute('height').replace( 'px', '')) # increase original height with branding bar's height finalSvg.setAttribute("width", str(width)) # same width as original figure finalSvg.setAttribute("height", str(height)) finalSvg.appendChild(figureSvg) # add the image finalSvg.appendChild(brandingSvg) # and the branding bar # Generate path where to store image with open(store_path, 'w') as dest: finalSvg.writexml(dest) # store to disk def _image_path(self, project_name, img_type): "Generate path where to store image" images_folder = self.file_helper.get_images_folder(project_name) file_name = FigureService._DEFAULT_IMAGE_FILE_NAME + img_type return utils.get_unique_file_name(images_folder, file_name) @staticmethod def _generate_image_name(project, user, operation, image_name): if not image_name: if operation is not None: # create a name based on the operation that created the image # e.g. TVB-Algo-Name-354 image_name = operation.algorithm.name.replace(' ', '-') else: # default to a generic name image_name = "figure" figure_count = dao.get_figure_count(project.id, user.id) + 1 return 'TVB-%s-%s' % (image_name, figure_count) def store_result_figure(self, project, user, img_type, export_data, image_name=None, operation_id=None): """ Store into a file, Result Image and reference in DB. """ store_path, file_name = self._image_path(project.name, img_type) if img_type == FigureService._TYPE_PNG: # PNG file from canvas self._write_png(store_path, export_data) elif img_type == FigureService._TYPE_SVG: # SVG file from svg viewer self._write_svg(store_path, export_data) if operation_id: operation = dao.get_operation_by_id(operation_id) else: operation = None operation_id = None image_name = self._generate_image_name(project, user, operation, image_name) # Store entity into DB entity = model.ResultFigure(operation_id, user.id, project.id, FigureService._DEFAULT_SESSION_NAME, image_name, file_name, img_type) entity = dao.store_entity(entity) # Load instance from DB to have lazy fields loaded figure = dao.load_figure(entity.id) # Write image meta data to disk self.file_helper.write_image_metadata(figure) if operation: # Force writing operation meta data on disk. # This is important later for operation import self.file_helper.write_operation_metadata(operation) def retrieve_result_figures(self, project, user, selected_session_name='all_sessions'): """ Retrieve from DB all the stored Displayer previews that belongs to the specified session. The previews are for current user and project; grouped by session. """ result, previews_info = dao.get_previews(project.id, user.id, selected_session_name) for name in result: for figure in result[name]: figures_folder = self.file_helper.get_images_folder( project.name) figure_full_path = os.path.join(figures_folder, figure.file_path) # Compute the path figure.file_path = utils.path2url_part(figure_full_path) return result, previews_info @staticmethod def load_figure(figure_id): """ Loads a stored figure by its id. """ return dao.load_figure(figure_id) def edit_result_figure(self, figure_id, **data): """ Retrieve and edit a previously stored figure. """ figure = dao.load_figure(figure_id) figure.session_name = data['session_name'] figure.name = data['name'] dao.store_entity(figure) # Load instance from DB to have lazy fields loaded. figure = dao.load_figure(figure_id) # Store figure meta data in an XML attached to the image. self.file_helper.write_image_metadata(figure) def remove_result_figure(self, figure_id): """ Remove figure from DB and file storage. """ figure = dao.load_figure(figure_id) # Delete all figure related files from disk. figures_folder = self.file_helper.get_images_folder( figure.project.name) path2figure = os.path.join(figures_folder, figure.file_path) if os.path.exists(path2figure): os.remove(path2figure) self.file_helper.remove_image_metadata(figure) # Remove figure reference from DB. result = dao.remove_entity(model.ResultFigure, figure_id) return result
class TestFigureService(TransactionalTestCase): """ Tests for the figure service """ def transactional_setup_method(self): self.figure_service = FigureService() self.user = TestFactory.create_user() self.project = TestFactory.create_project(admin=self.user) self.files_helper = FilesHelper() def transactional_teardown_method(self): self.delete_project_folders() def assertCanReadImage(self, image_path): try: Image.open(image_path).load() except (IOError, ValueError): raise AssertionError("Could not open %s as a image" % image_path) def store_test_png(self): self.figure_service.store_result_figure(self.project, self.user, "png", IMG_DATA, image_name="test-figure") def retrieve_images(self): figures_by_session, _ = self.figure_service.retrieve_result_figures(self.project, self.user) # flatten image session grouping figures = [] for fg in figures_by_session.itervalues(): figures.extend(fg) return figures def test_store_image(self): self.store_test_png() def test_store_image_from_operation(self): # test that image can be retrieved from operation test_operation = TestFactory.create_operation(test_user=self.user, test_project=self.project) self.figure_service.store_result_figure(self.project, self.user, "png", IMG_DATA, operation_id=test_operation.id) figures = dao.get_figures_for_operation(test_operation.id) assert 1 == len(figures) image_path = self.files_helper.get_images_folder(self.project.name) image_path = os.path.join(image_path, figures[0].file_path) self.assertCanReadImage(image_path) def test_store_and_retrieve_image(self): self.store_test_png() figures = self.retrieve_images() assert 1 == len(figures) image_path = utils.url2path(figures[0].file_path) self.assertCanReadImage(image_path) def test_load_figure(self): self.store_test_png() figures = self.retrieve_images() self.figure_service.load_figure(figures[0].id) def test_edit_figure(self): session_name = 'the altered ones' name = 'altered' self.store_test_png() figures = self.retrieve_images() self.figure_service.edit_result_figure(figures[0].id, session_name=session_name, name=name) figures_by_session, _ = self.figure_service.retrieve_result_figures(self.project, self.user) assert [session_name] == figures_by_session.keys() assert name == figures_by_session.values()[0][0].name def test_remove_figure(self): self.store_test_png() figures = self.retrieve_images() assert 1 == len(figures) self.figure_service.remove_result_figure(figures[0].id) figures = self.retrieve_images() assert 0 == len(figures)
def __init__(self): self.logger = get_logger(self.__class__.__module__) self.file_helper = FilesHelper()
def launch(self, uploaded, rotate_x=0, rotate_y=0, rotate_z=0): """ Execute import operations: unpack ZIP and build Connectivity object as result. :param uploaded: an archive containing the Connectivity data to be imported :returns: `Connectivity` :raises LaunchException: when `uploaded` is empty or nonexistent :raises Exception: when * weights or tracts matrix is invalid (negative values, wrong shape) * any of the vector orientation, areas, cortical or hemisphere is \ different from the expected number of nodes """ if uploaded is None: raise LaunchException("Please select ZIP file which contains data to import") files = FilesHelper().unpack_zip(uploaded, self.storage_path) weights_matrix = None centres = None labels_vector = None tract_matrix = None orientation = None areas = None cortical_vector = None hemisphere_vector = None for file_name in files: if file_name.lower().find(self.WEIGHT_TOKEN) >= 0: weights_matrix = read_list_data(file_name) continue if file_name.lower().find(self.POSITION_TOKEN) >= 0: centres = read_list_data(file_name, skiprows=1, usecols=[1, 2, 3]) labels_vector = read_list_data(file_name, dtype=numpy.str, skiprows=1, usecols=[0]) continue if file_name.lower().find(self.TRACT_TOKEN) >= 0: tract_matrix = read_list_data(file_name) continue if file_name.lower().find(self.ORIENTATION_TOKEN) >= 0: orientation = read_list_data(file_name) continue if file_name.lower().find(self.AREA_TOKEN) >= 0: areas = read_list_data(file_name) continue if file_name.lower().find(self.CORTICAL_INFO) >= 0: cortical_vector = read_list_data(file_name, dtype=numpy.bool) continue if file_name.lower().find(self.HEMISPHERE_INFO) >= 0: hemisphere_vector = read_list_data(file_name, dtype=numpy.bool) continue ### Clean remaining text-files. FilesHelper.remove_files(files, True) result = Connectivity() result.storage_path = self.storage_path result.nose_correction = [rotate_x, rotate_y, rotate_z] ### Fill positions if centres is None: raise Exception("Positions for Connectivity Regions are required! " "We expect a file *position* inside the uploaded ZIP.") expected_number_of_nodes = len(centres) if expected_number_of_nodes < 2: raise Exception("A connectivity with at least 2 nodes is expected") result.centres = centres if labels_vector is not None: result.region_labels = labels_vector ### Fill and check weights if weights_matrix is not None: if numpy.any([x < 0 for x in weights_matrix.flatten()]): raise Exception("Negative values are not accepted in weights matrix! " "Please check your file, and use values >= 0") if weights_matrix.shape != (expected_number_of_nodes, expected_number_of_nodes): raise Exception("Unexpected shape for weights matrix! " "Should be %d x %d " % (expected_number_of_nodes, expected_number_of_nodes)) result.weights = weights_matrix ### Fill and check tracts if tract_matrix is not None: if numpy.any([x < 0 for x in tract_matrix.flatten()]): raise Exception("Negative values are not accepted in tracts matrix! " "Please check your file, and use values >= 0") if tract_matrix.shape != (expected_number_of_nodes, expected_number_of_nodes): raise Exception("Unexpected shape for tracts matrix! " "Should be %d x %d " % (expected_number_of_nodes, expected_number_of_nodes)) result.tract_lengths = tract_matrix if orientation is not None: if len(orientation) != expected_number_of_nodes: raise Exception("Invalid size for vector orientation. " "Expected the same as region-centers number %d" % expected_number_of_nodes) result.orientations = orientation if areas is not None: if len(areas) != expected_number_of_nodes: raise Exception("Invalid size for vector areas. " "Expected the same as region-centers number %d" % expected_number_of_nodes) result.areas = areas if cortical_vector is not None: if len(cortical_vector) != expected_number_of_nodes: raise Exception("Invalid size for vector cortical. " "Expected the same as region-centers number %d" % expected_number_of_nodes) result.cortical = cortical_vector if hemisphere_vector is not None: if len(hemisphere_vector) != expected_number_of_nodes: raise Exception("Invalid size for vector hemispheres. " "Expected the same as region-centers number %d" % expected_number_of_nodes) result.hemispheres = hemisphere_vector return result
class SimulatorController(BurstBaseController): ACTION_KEY = 'action' PREVIOUS_ACTION_KEY = 'previous_action' FORM_KEY = 'form' IS_MODEL_FRAGMENT_KEY = 'is_model_fragment' IS_SURFACE_SIMULATION_KEY = 'is_surface_simulation' IS_FIRST_FRAGMENT_KEY = 'is_first_fragment' IS_LAST_FRAGMENT_KEY = 'is_last_fragment' IS_COPY = 'sim_copy' IS_LOAD = 'sim_load' dict_to_render = { ACTION_KEY: None, PREVIOUS_ACTION_KEY: None, FORM_KEY: None, IS_MODEL_FRAGMENT_KEY: False, IS_SURFACE_SIMULATION_KEY: False, IS_FIRST_FRAGMENT_KEY: False, IS_LAST_FRAGMENT_KEY: False, IS_COPY: False, IS_LOAD: False } def __init__(self): BurstBaseController.__init__(self) self.range_parameters = SimulatorRangeParameters() self.burst_service2 = BurstService2() self.simulator_service = SimulatorService() self.files_helper = FilesHelper() self.cached_simulator_algorithm = self.flow_service.get_algorithm_by_module_and_class( IntrospectionRegistry.SIMULATOR_MODULE, IntrospectionRegistry.SIMULATOR_CLASS) @expose_page @settings @context_selected def index(self): """Get on burst main page""" template_specification = dict( mainContent="burst/main_burst", title="Simulation Cockpit", baseUrl=TvbProfile.current.web.BASE_URL, includedResources='project/included_resources') project = common.get_current_project() burst_config = BurstConfiguration2(project.id) common.add2session(common.KEY_BURST_CONFIG, burst_config) template_specification['burstConfig'] = burst_config template_specification[ 'burst_list'] = self.burst_service2.get_available_bursts( common.get_current_project().id) portlets_list = [] # self.burst_service.get_available_portlets() template_specification['portletList'] = portlets_list template_specification['selectedPortlets'] = json.dumps(portlets_list) form = self.prepare_first_fragment() dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.IS_FIRST_FRAGMENT_KEY] = True dict_to_render[self.FORM_KEY] = form dict_to_render[self.ACTION_KEY] = "/burst/set_connectivity" template_specification.update(**dict_to_render) return self.fill_default_attributes(template_specification) def prepare_first_fragment(self): is_simulator_copy = common.get_from_session( common.KEY_IS_SIMULATOR_COPY) if is_simulator_copy is None: is_simulator_copy = False adapter_instance = ABCAdapter.build_adapter( self.cached_simulator_algorithm) form = adapter_instance.get_form()('', common.get_current_project().id, is_simulator_copy) session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) if session_stored_simulator is None: session_stored_simulator = Simulator( ) # self.burst_service.new_burst_configuration(common.get_current_project().id) common.add2session(common.KEY_SIMULATOR_CONFIG, session_stored_simulator) form.fill_from_trait(session_stored_simulator) return form @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def set_connectivity(self, **data): session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) is_simulator_copy = common.get_from_session( common.KEY_IS_SIMULATOR_COPY) is_simulator_load = common.get_from_session( common.KEY_IS_SIMULATOR_LOAD) form = SimulatorAdapterForm() if cherrypy.request.method == 'POST': is_simulator_copy = False form.fill_from_post(data) connectivity_index_gid = form.connectivity.value conduction_speed = form.conduction_speed.value coupling = form.coupling.value connectivity_index = ABCAdapter.load_entity_by_gid( connectivity_index_gid) connectivity = h5.load_from_index(connectivity_index) # TODO: handle this cases in a better manner session_stored_simulator.connectivity = connectivity session_stored_simulator.conduction_speed = conduction_speed session_stored_simulator.coupling = coupling() next_form = get_form_for_coupling( type(session_stored_simulator.coupling))() self.range_parameters.coupling_parameters = next_form.get_range_parameters( ) next_form.fill_from_trait(session_stored_simulator.coupling) dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = next_form dict_to_render[self.ACTION_KEY] = '/burst/set_coupling_params' dict_to_render[self.PREVIOUS_ACTION_KEY] = '/burst/set_connectivity' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load return dict_to_render @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def set_coupling_params(self, **data): session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) is_simulator_copy = common.get_from_session( common.KEY_IS_SIMULATOR_COPY) is_simulator_load = common.get_from_session( common.KEY_IS_SIMULATOR_LOAD) if cherrypy.request.method == 'POST': is_simulator_copy = False form = get_form_for_coupling( type(session_stored_simulator.coupling))() form.fill_from_post(data) form.fill_trait(session_stored_simulator.coupling) surface_fragment = SimulatorSurfaceFragment( '', common.get_current_project().id) surface_fragment.fill_from_trait(session_stored_simulator) dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = surface_fragment dict_to_render[self.ACTION_KEY] = '/burst/set_surface' dict_to_render[self.PREVIOUS_ACTION_KEY] = '/burst/set_coupling_params' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load return dict_to_render @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def set_surface(self, **data): session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) is_simulator_copy = common.get_from_session( common.KEY_IS_SIMULATOR_COPY) is_simulator_load = common.get_from_session( common.KEY_IS_SIMULATOR_LOAD) dict_to_render = copy.deepcopy(self.dict_to_render) surface_index = None if cherrypy.request.method == 'POST': is_simulator_copy = False form = SimulatorSurfaceFragment() form.fill_from_post(data) surface_index_gid = form.surface.value # surface_index_gid = data['_surface'] if surface_index_gid is None: session_stored_simulator.surface = None else: surface_index = ABCAdapter.load_entity_by_gid( surface_index_gid) session_stored_simulator.surface = Cortex() if session_stored_simulator.surface is None: stimuli_fragment = SimulatorStimulusFragment( '', common.get_current_project().id, False) stimuli_fragment.fill_from_trait(session_stored_simulator) dict_to_render[self.FORM_KEY] = stimuli_fragment dict_to_render[self.ACTION_KEY] = '/burst/set_stimulus' dict_to_render[self.PREVIOUS_ACTION_KEY] = '/burst/set_surface' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load return dict_to_render # TODO: work-around this situation: surf_index filter rm_fragment = SimulatorRMFragment('', common.get_current_project().id, surface_index) rm_fragment.fill_from_trait(session_stored_simulator) dict_to_render[self.FORM_KEY] = rm_fragment dict_to_render[self.ACTION_KEY] = '/burst/set_cortex' dict_to_render[self.PREVIOUS_ACTION_KEY] = '/burst/set_surface' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load return dict_to_render @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def set_cortex(self, **data): session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) is_simulator_copy = common.get_from_session( common.KEY_IS_SIMULATOR_COPY) is_simulator_load = common.get_from_session( common.KEY_IS_SIMULATOR_LOAD) if cherrypy.request.method == 'POST': is_simulator_copy = False rm_fragment = SimulatorRMFragment() rm_fragment.fill_from_post(data) session_stored_simulator.surface.coupling_strength = rm_fragment.coupling_strength.data lc_gid = rm_fragment.lc.value if lc_gid == 'None': lc_index = ABCAdapter.load_entity_by_gid(lc_gid) lc = h5.load_from_index(lc_index) session_stored_simulator.surface.local_connectivity = lc rm_gid = rm_fragment.rm.value rm_index = ABCAdapter.load_entity_by_gid(rm_gid) rm = h5.load_from_index(rm_index) session_stored_simulator.surface.region_mapping_data = rm stimuli_fragment = SimulatorStimulusFragment( '', common.get_current_project().id, True) stimuli_fragment.fill_from_trait(session_stored_simulator) dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = stimuli_fragment dict_to_render[self.ACTION_KEY] = '/burst/set_stimulus' dict_to_render[self.PREVIOUS_ACTION_KEY] = '/burst/set_cortex' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load return dict_to_render @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def set_stimulus(self, **data): session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) is_simulator_copy = common.get_from_session( common.KEY_IS_SIMULATOR_COPY) is_simulator_load = common.get_from_session( common.KEY_IS_SIMULATOR_LOAD) if cherrypy.request.method == 'POST': is_simulator_copy = False stimuli_fragment = SimulatorStimulusFragment( '', common.get_current_project().id, session_stored_simulator.is_surface_simulation) stimuli_fragment.fill_from_post(data) stimulus_gid = stimuli_fragment.stimulus.value if stimulus_gid != None: stimulus_index = ABCAdapter.load_entity_by_gid(stimulus_gid) stimulus = h5.load_from_index(stimulus_index) session_stored_simulator.stimulus = stimulus model_fragment = SimulatorModelFragment( '', common.get_current_project().id) model_fragment.fill_from_trait(session_stored_simulator) dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = model_fragment dict_to_render[self.ACTION_KEY] = '/burst/set_model' dict_to_render[self.PREVIOUS_ACTION_KEY] = '/burst/set_stimulus' dict_to_render[self.IS_MODEL_FRAGMENT_KEY] = True dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load dict_to_render[ self. IS_SURFACE_SIMULATION_KEY] = session_stored_simulator.is_surface_simulation return dict_to_render @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def set_model(self, **data): session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) is_simulator_copy = common.get_from_session( common.KEY_IS_SIMULATOR_COPY) is_simulator_load = common.get_from_session( common.KEY_IS_SIMULATOR_LOAD) if cherrypy.request.method == 'POST': is_simulator_copy = False form = SimulatorModelFragment() form.fill_from_post(data) session_stored_simulator.model = form.model.value() form = get_form_for_model(type(session_stored_simulator.model))() self.range_parameters.model_parameters = form.get_range_parameters() form.fill_from_trait(session_stored_simulator.model) dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = form dict_to_render[self.ACTION_KEY] = '/burst/set_model_params' dict_to_render[self.PREVIOUS_ACTION_KEY] = '/burst/set_model' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load return dict_to_render @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def set_model_params(self, **data): session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) is_simulator_copy = common.get_from_session( common.KEY_IS_SIMULATOR_COPY) is_simulator_load = common.get_from_session( common.KEY_IS_SIMULATOR_LOAD) if cherrypy.request.method == 'POST': is_simulator_copy = False form = get_form_for_model(type(session_stored_simulator.model))() form.fill_from_post(data) form.fill_trait(session_stored_simulator.model) integrator_fragment = SimulatorIntegratorFragment( '', common.get_current_project().id) integrator_fragment.fill_from_trait(session_stored_simulator) dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = integrator_fragment dict_to_render[self.ACTION_KEY] = '/burst/set_integrator' dict_to_render[self.PREVIOUS_ACTION_KEY] = '/burst/set_model_params' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load return dict_to_render # TODO: add state_variables selection step # @cherrypy.expose # @using_jinja_template("wizzard_form") # @handle_error(redirect=False) # @check_user # def set_model_variables_to_monitor(self, data): # session_stored_simulator = common.get_from_session(common.KEY_SIMULATOR_CONFIG) # form = get_form_for_model(type(session_stored_simulator.model.variables))() # form.fill_from_post(data) # # form.fill_trait(session_stored_simulator.model) # # integrator_fragment = SimulatorIntegratorFragment('', common.get_current_project().id) # # return {'form': integrator_fragment, 'action': '/burst/set_integrator', # 'previous_action': '/burst/set_model_variables_to_monitor'} @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def set_integrator(self, **data): session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) is_simulator_copy = common.get_from_session( common.KEY_IS_SIMULATOR_COPY) is_simulator_load = common.get_from_session( common.KEY_IS_SIMULATOR_LOAD) if cherrypy.request.method == 'POST': is_simulator_copy = False fragment = SimulatorIntegratorFragment() fragment.fill_from_post(data) session_stored_simulator.integrator = fragment.integrator.value() form = get_form_for_integrator( type(session_stored_simulator.integrator))() form.fill_from_trait(session_stored_simulator.integrator) dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = form dict_to_render[self.ACTION_KEY] = '/burst/set_integrator_params' dict_to_render[self.PREVIOUS_ACTION_KEY] = '/burst/set_integrator' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load return dict_to_render @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def set_integrator_params(self, **data): session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) is_simulator_copy = common.get_from_session( common.KEY_IS_SIMULATOR_COPY) is_simulator_load = common.get_from_session( common.KEY_IS_SIMULATOR_LOAD) if cherrypy.request.method == 'POST': is_simulator_copy = False form = get_form_for_integrator( type(session_stored_simulator.integrator))() form.fill_from_post(data) form.fill_trait(session_stored_simulator.integrator) if isinstance(session_stored_simulator.integrator, IntegratorStochastic): integrator_noise_fragment = get_form_for_noise( type(session_stored_simulator.integrator.noise))() self.range_parameters.integrator_noise_parameters = integrator_noise_fragment.get_range_parameters( ) integrator_noise_fragment.fill_from_trait( session_stored_simulator.integrator.noise) dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = integrator_noise_fragment dict_to_render[self.ACTION_KEY] = '/burst/set_noise_params' dict_to_render[ self.PREVIOUS_ACTION_KEY] = '/burst/set_integrator_params' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load return dict_to_render monitor_fragment = SimulatorMonitorFragment( '', common.get_current_project().id) monitor_fragment.fill_from_trait(session_stored_simulator) dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = monitor_fragment dict_to_render[self.ACTION_KEY] = '/burst/set_monitors' dict_to_render[ self.PREVIOUS_ACTION_KEY] = '/burst/set_integrator_params' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load return dict_to_render @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def set_noise_params(self, **data): session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) is_simulator_copy = common.get_from_session( common.KEY_IS_SIMULATOR_COPY) is_simulator_load = common.get_from_session( common.KEY_IS_SIMULATOR_LOAD) if cherrypy.request.method == 'POST': is_simulator_copy = False form = get_form_for_noise( type(session_stored_simulator.integrator.noise))() form.fill_from_post(data) form.fill_trait(session_stored_simulator.integrator.noise) if isinstance(session_stored_simulator.integrator.noise, Additive): monitor_fragment = SimulatorMonitorFragment( '', common.get_current_project().id) monitor_fragment.fill_from_trait(session_stored_simulator) dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = monitor_fragment dict_to_render[self.ACTION_KEY] = '/burst/set_monitors' dict_to_render[ self.PREVIOUS_ACTION_KEY] = '/burst/set_noise_params' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load return dict_to_render equation_form = get_form_for_equation( type(session_stored_simulator.integrator.noise.b))() equation_form.equation.data = session_stored_simulator.integrator.noise.b.__class__.__name__ dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = equation_form dict_to_render[self.ACTION_KEY] = '/burst/set_noise_equation_params' dict_to_render[self.PREVIOUS_ACTION_KEY] = '/burst/set_noise_params' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load return dict_to_render @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def set_noise_equation_params(self, **data): session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) is_simulator_copy = common.get_from_session( common.KEY_IS_SIMULATOR_COPY) is_simulator_load = common.get_from_session( common.KEY_IS_SIMULATOR_LOAD) if cherrypy.request.method == 'POST': is_simulator_copy = False form = get_form_for_equation( type(session_stored_simulator.integrator.noise.b))() form.fill_from_post(data) form.fill_trait(session_stored_simulator.integrator.noise.b) monitor_fragment = SimulatorMonitorFragment( '', common.get_current_project().id) monitor_fragment.fill_from_trait(session_stored_simulator) dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = monitor_fragment dict_to_render[self.ACTION_KEY] = '/burst/set_monitors' dict_to_render[ self.PREVIOUS_ACTION_KEY] = '/burst/set_noise_equation_params' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load return dict_to_render @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def set_monitors(self, **data): session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) is_simulator_copy = common.get_from_session( common.KEY_IS_SIMULATOR_COPY) is_simulator_load = common.get_from_session( common.KEY_IS_SIMULATOR_LOAD) if cherrypy.request.method == 'POST': is_simulator_copy = False # TODO: handle multiple monitors fragment = SimulatorMonitorFragment() fragment.fill_from_post(data) session_stored_simulator.monitors = [fragment.monitor.value()] monitor = session_stored_simulator.monitors[0] form = get_form_for_monitor(type(monitor))( '', common.get_current_project().id) form.fill_from_trait(monitor) dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = form dict_to_render[self.ACTION_KEY] = '/burst/set_monitor_params' dict_to_render[self.PREVIOUS_ACTION_KEY] = '/burst/set_monitors' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load return dict_to_render @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def set_monitor_params(self, **data): session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) monitor = session_stored_simulator.monitors[0] is_simulator_copy = common.get_from_session( common.KEY_IS_SIMULATOR_COPY) is_simulator_load = common.get_from_session( common.KEY_IS_SIMULATOR_LOAD) if cherrypy.request.method == 'POST': is_simulator_copy = False form = get_form_for_monitor(type(monitor))() form.fill_from_post(data) form.fill_trait(monitor) if isinstance(monitor, Bold): next_form = get_form_for_equation(type(monitor.hrf_kernel))() next_form.fill_from_trait( session_stored_simulator.monitors[0].hrf_kernel) dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = next_form dict_to_render[self.ACTION_KEY] = '/burst/set_monitor_equation' dict_to_render[ self.PREVIOUS_ACTION_KEY] = '/burst/set_monitor_params' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load return dict_to_render session_stored_simulator.monitors = [monitor] next_form = SimulatorLengthFragment() next_form.fill_from_trait(session_stored_simulator) dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = next_form dict_to_render[self.ACTION_KEY] = '/burst/set_simulation_length' dict_to_render[self.PREVIOUS_ACTION_KEY] = '/burst/set_monitor_params' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load if is_simulator_load: dict_to_render[self.ACTION_KEY] = '' dict_to_render[self.IS_LAST_FRAGMENT_KEY] = True return dict_to_render @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def set_monitor_equation(self, **data): session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) is_simulator_copy = common.get_from_session( common.KEY_IS_SIMULATOR_COPY) is_simulator_load = common.get_from_session( common.KEY_IS_SIMULATOR_LOAD) if cherrypy.request.method == 'POST': is_simulator_copy = False form = get_form_for_monitor( type(session_stored_simulator.monitors[0]))() form.fill_from_post(data) form.fill_trait(session_stored_simulator.monitor.hrf_kernel) next_form = SimulatorLengthFragment() dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = next_form dict_to_render[self.ACTION_KEY] = '/burst/set_simulation_length' dict_to_render[ self.PREVIOUS_ACTION_KEY] = '/burst/set_monitor_equation' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LOAD] = is_simulator_load if is_simulator_load: dict_to_render[self.ACTION_KEY] = '' dict_to_render[self.IS_LAST_FRAGMENT_KEY] = True return dict_to_render @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def set_simulation_length(self, **data): session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) is_simulator_copy = common.get_from_session( common.KEY_IS_SIMULATOR_COPY) is_simulator_load = common.get_from_session( common.KEY_IS_SIMULATOR_LOAD) session_burst_config = common.get_from_session(common.KEY_BURST_CONFIG) dict_to_render = copy.deepcopy(self.dict_to_render) if is_simulator_load: common.add2session(common.KEY_IS_SIMULATOR_LOAD, False) next_form = SimulatorFinalFragment() if session_burst_config.name: burst_name = session_burst_config.name copy_prefix = 'Copy of ' if is_simulator_copy and burst_name.find(copy_prefix) < 0: burst_name = copy_prefix + burst_name next_form.simulation_name.data = burst_name if cherrypy.request.method == 'POST': is_simulator_copy = False fragment = SimulatorLengthFragment() fragment.fill_from_post(data) session_stored_simulator.simulation_length = fragment.length.value dict_to_render[self.FORM_KEY] = next_form dict_to_render[self.ACTION_KEY] = '/burst/setup_pse' dict_to_render[ self.PREVIOUS_ACTION_KEY] = '/burst/set_simulation_length' dict_to_render[self.IS_COPY] = is_simulator_copy dict_to_render[self.IS_LAST_FRAGMENT_KEY] = True return dict_to_render @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def setup_pse(self, **data): next_form = SimulatorPSEConfigurationFragment( self.range_parameters.get_all_range_parameters()) dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = next_form dict_to_render[self.ACTION_KEY] = '/burst/set_pse_params' dict_to_render[ self.PREVIOUS_ACTION_KEY] = '/burst/set_simulation_length' return dict_to_render @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def set_pse_params(self, **data): form = SimulatorPSEConfigurationFragment( self.range_parameters.get_all_range_parameters()) form.fill_from_post(data) param1 = form.pse_param1.value param2 = None if not form.pse_param2.value == form.pse_param2.missing_value: param2 = form.pse_param2.value project_id = common.get_current_project().id next_form = SimulatorPSEParamRangeFragment(param1, param2, project_id=project_id) dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.FORM_KEY] = next_form dict_to_render[self.ACTION_KEY] = '/burst/launch_pse' dict_to_render[self.PREVIOUS_ACTION_KEY] = '/burst/set_pse_params' return dict_to_render @cherrypy.expose @handle_error(redirect=False) @check_user def launch_pse(self, **data): # TODO: Split into: set range values and Launch, show message with finished config and nr of simulations all_range_parameters = self.range_parameters.get_all_range_parameters() range_param1, range_param2 = SimulatorPSEParamRangeFragment.fill_from_post( all_range_parameters, **data) session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) project = common.get_current_project() user = common.get_logged_user() burst_config = common.get_from_session(common.KEY_BURST_CONFIG) burst_config.start_time = datetime.now() # if burst_name != 'none_undefined': # burst_config.name = burst_name # TODO: branch simulation name is different if burst_config.name is None: new_id = dao.get_max_burst_id() + 1 burst_config.name = 'simulation_' + str(new_id) operation_group = OperationGroup( project.id, ranges=[range_param1.to_json(), range_param2.to_json()]) operation_group = dao.store_entity(operation_group) metric_operation_group = OperationGroup( project.id, ranges=[range_param1.to_json(), range_param2.to_json()]) metric_operation_group = dao.store_entity(metric_operation_group) burst_config.operation_group = operation_group burst_config.operation_group_id = operation_group.id burst_config.metric_operation_group = metric_operation_group burst_config.metric_operation_group_id = metric_operation_group.id dao.store_entity(burst_config) try: thread = threading.Thread( target=self.simulator_service.async_launch_and_prepare_pse, kwargs={ 'burst_config': burst_config, 'user': user, 'project': project, 'simulator_algo': self.cached_simulator_algorithm, 'range_param1': range_param1, 'range_param2': range_param2, 'session_stored_simulator': session_stored_simulator }) thread.start() except BurstServiceException as e: self.logger.exception("Could not launch burst!") return {'error': e.message} @expose_json def launch_simulation(self, launch_mode, **data): current_form = SimulatorFinalFragment() try: current_form.fill_from_post(data) except Exception as exc: self.logger.exception(exc) return {'error': str(exc)} burst_name = current_form.simulation_name.value session_stored_simulator = common.get_from_session( common.KEY_SIMULATOR_CONFIG) is_simulator_copy = common.get_from_session( common.KEY_IS_SIMULATOR_COPY) project = common.get_current_project() user = common.get_logged_user() session_burst_config = common.get_from_session(common.KEY_BURST_CONFIG) if burst_name != 'none_undefined': session_burst_config.name = burst_name burst_config_to_store = session_burst_config simulation_state_index_gid = None if launch_mode == self.simulator_service.LAUNCH_NEW: if session_burst_config.name is None: new_id = dao.get_max_burst_id() + 1 session_burst_config.name = 'simulation_' + str(new_id) if is_simulator_copy: burst_config_to_store = session_burst_config.clone() else: burst_config_to_store = session_burst_config.clone() count = dao.count_bursts_with_name(session_burst_config.name, session_burst_config.project_id) session_burst_config.name = session_burst_config.name + "_" + launch_mode + str( count) simulation_state_index = dao.get_generic_entity( SimulationStateIndex.__module__ + "." + SimulationStateIndex.__name__, session_burst_config.id, "fk_parent_burst") if simulation_state_index is None or len( simulation_state_index) < 1: exc = BurstServiceException( "Simulation State not found for %s, thus we are unable to branch from " "it!" % session_burst_config.name) self.logger.error(exc) raise exc simulation_state_index_gid = simulation_state_index[0].gid burst_config_to_store.start_time = datetime.now() dao.store_entity(burst_config_to_store) try: thread = threading.Thread(target=self.simulator_service. async_launch_and_prepare_simulation, kwargs={ 'burst_config': burst_config_to_store, 'user': user, 'project': project, 'simulator_algo': self.cached_simulator_algorithm, 'session_stored_simulator': session_stored_simulator, 'simulation_state_gid': simulation_state_index_gid }) thread.start() return {'id': burst_config_to_store.id} except BurstServiceException as e: self.logger.exception('Could not launch burst!') return {'error': e.message} @expose_fragment('burst/burst_history') def load_burst_history(self): """ Load the available burst that are stored in the database at this time. This is one alternative to 'chrome-back problem'. """ session_burst = common.get_from_session(common.KEY_BURST_CONFIG) bursts = self.burst_service2.get_available_bursts( common.get_current_project().id) self.burst_service2.populate_burst_disk_usage(bursts) return {'burst_list': bursts, 'selectedBurst': session_burst.id} @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def load_burst_read_only(self, burst_config_id): try: burst_config = dao.get_burst_by_id(burst_config_id) common.add2session(common.KEY_BURST_CONFIG, burst_config) simulator_index = dao.get_generic_entity(SimulatorIndex, burst_config.id, 'fk_parent_burst')[0] simulator_gid = simulator_index.gid project = common.get_current_project() storage_path = self.files_helper.get_project_folder( project, str(simulator_index.fk_from_operation)) simulator, _, _ = self.simulator_service.deserialize_simulator( simulator_gid, storage_path) session_stored_simulator = simulator common.add2session(common.KEY_SIMULATOR_CONFIG, session_stored_simulator) common.add2session(common.KEY_IS_SIMULATOR_LOAD, True) common.add2session(common.KEY_IS_SIMULATOR_COPY, False) form = self.prepare_first_fragment() dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.IS_FIRST_FRAGMENT_KEY] = True dict_to_render[self.FORM_KEY] = form dict_to_render[self.ACTION_KEY] = "/burst/set_connectivity" dict_to_render[self.IS_LOAD] = True return dict_to_render except Exception: ### Most probably Burst was removed. Delete it from session, so that client ### has a good chance to get a good response on refresh self.logger.exception("Error loading burst") common.remove_from_session(common.KEY_BURST_CONFIG) raise @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def copy_simulator_configuration(self, burst_config_id): burst_config = dao.get_burst_by_id(burst_config_id) common.add2session(common.KEY_BURST_CONFIG, burst_config) simulator_index = dao.get_generic_entity(SimulatorIndex, burst_config.id, 'fk_parent_burst')[0] simulator_gid = simulator_index.gid project = common.get_current_project() storage_path = self.files_helper.get_project_folder( project, str(simulator_index.fk_from_operation)) simulator, _, _ = self.simulator_service.deserialize_simulator( simulator_gid, storage_path) session_stored_simulator = simulator common.add2session(common.KEY_SIMULATOR_CONFIG, session_stored_simulator) common.add2session(common.KEY_IS_SIMULATOR_COPY, True) common.add2session(common.KEY_IS_SIMULATOR_LOAD, False) form = self.prepare_first_fragment() dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.IS_FIRST_FRAGMENT_KEY] = True dict_to_render[self.FORM_KEY] = form dict_to_render[self.ACTION_KEY] = "/burst/set_connectivity" dict_to_render[self.IS_COPY] = True return dict_to_render @cherrypy.expose @using_jinja_template("wizzard_form") @handle_error(redirect=False) @check_user def reset_simulator_configuration(self): common.add2session(common.KEY_SIMULATOR_CONFIG, None) common.add2session(common.KEY_IS_SIMULATOR_COPY, False) common.add2session(common.KEY_IS_SIMULATOR_LOAD, False) project = common.get_current_project() common.add2session(common.KEY_BURST_CONFIG, BurstConfiguration2(project.id)) form = self.prepare_first_fragment() dict_to_render = copy.deepcopy(self.dict_to_render) dict_to_render[self.IS_FIRST_FRAGMENT_KEY] = True dict_to_render[self.FORM_KEY] = form dict_to_render[self.ACTION_KEY] = "/burst/set_connectivity" return dict_to_render @expose_json def rename_burst(self, burst_id, burst_name): """ Rename the burst given by burst_id, setting it's new name to burst_name. """ validation_result = SimulatorFinalFragment.is_burst_name_ok(burst_name) if validation_result is True: self.burst_service2.rename_burst(burst_id, burst_name) return {'success': "Simulation successfully renamed!"} else: self.logger.exception(validation_result) return {'error': validation_result} @expose_json def get_history_status(self, **data): """ For each burst id received, get the status and return it. """ return self.burst_service2.update_history_status( json.loads(data['burst_ids']))
class BurstService(object): LAUNCH_NEW = 'new' LAUNCH_BRANCH = 'branch' def __init__(self): self.logger = get_logger(self.__class__.__module__) self.files_helper = FilesHelper() def mark_burst_finished(self, burst_entity, burst_status=None, error_message=None): """ Mark Burst status field. Also compute 'weight' for current burst: no of operations inside, estimate time on disk... :param burst_entity: BurstConfiguration to be updated, at finish time. :param burst_status: BurstConfiguration status. By default BURST_FINISHED :param error_message: If given, set the status to error and perpetuate the message. """ if burst_status is None: burst_status = BurstConfiguration.BURST_FINISHED if error_message is not None: burst_status = BurstConfiguration.BURST_ERROR try: # If there are any DataType Groups in current Burst, update their counter. burst_dt_groups = dao.get_generic_entity(DataTypeGroup, burst_entity.gid, "fk_parent_burst") for dt_group in burst_dt_groups: dt_group.count_results = dao.count_datatypes_in_group( dt_group.id) dt_group.disk_size, dt_group.subject = dao.get_summary_for_group( dt_group.id) dao.store_entity(dt_group) # Update actual Burst entity fields burst_entity.datatypes_number = dao.count_datatypes_in_burst( burst_entity.gid) burst_entity.status = burst_status burst_entity.error_message = error_message burst_entity.finish_time = datetime.now() dao.store_entity(burst_entity) self.update_burst_configuration_h5(burst_entity) except Exception: self.logger.exception( "Could not correctly update Burst status and meta-data!") burst_entity.status = burst_status burst_entity.error_message = "Error when updating Burst Status" burst_entity.finish_time = datetime.now() dao.store_entity(burst_entity) self.update_burst_configuration_h5(burst_entity) def persist_operation_state(self, operation, operation_status, message=None): """ Update Operation instance state. Store it in DB and on HDD/ :param operation: Operation instance :param operation_status: new status :param message: message in case of error :return: operation instance changed """ operation.mark_complete(operation_status, message) operation = dao.store_entity(operation) # update burst also burst_config = self.get_burst_for_operation_id(operation.id) if burst_config is not None: burst_status = STATUS_FOR_OPERATION.get(operation_status) self.mark_burst_finished(burst_config, burst_status, message) return operation @staticmethod def get_burst_for_operation_id(operation_id, is_group=False): return dao.get_burst_for_operation_id(operation_id, is_group) def rename_burst(self, burst_id, new_name): """ Rename the burst given by burst_id, setting it's new name to burst_name. """ burst = dao.get_burst_by_id(burst_id) burst.name = new_name dao.store_entity(burst) self.update_burst_configuration_h5(burst) @staticmethod def get_available_bursts(project_id): """ Return all the burst for the current project. """ bursts = dao.get_bursts_for_project( project_id, page_size=MAX_BURSTS_DISPLAYED) or [] return bursts @staticmethod def populate_burst_disk_usage(bursts): """ Adds a disk_usage field to each burst object. The disk usage is computed as the sum of the datatypes generated by a burst """ sizes = dao.compute_bursts_disk_size([b.gid for b in bursts]) for b in bursts: b.disk_size = format_bytes_human(sizes[b.gid]) def update_history_status(self, id_list): """ For each burst_id received in the id_list read new status from DB and return a list [id, new_status, is_group, message, running_time] tuple. """ result = [] for b_id in id_list: burst = dao.get_burst_by_id(b_id) if burst is not None: if burst.status == burst.BURST_RUNNING: running_time = datetime.now() - burst.start_time else: running_time = burst.finish_time - burst.start_time running_time = format_timedelta(running_time, most_significant2=False) if burst.status == burst.BURST_ERROR: msg = 'Check Operations page for error Message' else: msg = '' result.append([ burst.id, burst.status, burst.is_group, msg, running_time ]) else: self.logger.debug("Could not find burst with id=" + str(b_id) + ". Might have been deleted by user!!") return result @staticmethod def update_simulation_fields(burst, op_simulation_id, simulation_gid): burst.fk_simulation = op_simulation_id burst.simulator_gid = simulation_gid.hex burst = dao.store_entity(burst) return burst def update_burst_configuration_h5(self, burst_configuration): # type: (BurstConfiguration) -> None project = dao.get_project_by_id(burst_configuration.fk_project) storage_path = self.files_helper.get_project_folder( project, str(burst_configuration.fk_simulation)) self.store_burst_configuration(burst_configuration, storage_path) @staticmethod def load_burst_configuration(burst_config_id): # type: (int) -> BurstConfiguration burst_config = dao.get_burst_by_id(burst_config_id) return burst_config @staticmethod def prepare_burst_for_pse(burst_config): # type: (BurstConfiguration) -> (BurstConfiguration) operation_group = OperationGroup(burst_config.fk_project, ranges=burst_config.ranges) operation_group = dao.store_entity(operation_group) metric_operation_group = OperationGroup(burst_config.fk_project, ranges=burst_config.ranges) metric_operation_group = dao.store_entity(metric_operation_group) burst_config.operation_group = operation_group burst_config.fk_operation_group = operation_group.id burst_config.metric_operation_group = metric_operation_group burst_config.fk_metric_operation_group = metric_operation_group.id return dao.store_entity(burst_config) @staticmethod def store_burst_configuration(burst_config, storage_path): bc_path = h5.path_for(storage_path, BurstConfigurationH5, burst_config.gid) with BurstConfigurationH5(bc_path) as bc_h5: bc_h5.store(burst_config) @staticmethod def load_burst_configuration_from_folder(simulator_folder, project): bc_h5_filename = DirLoader( simulator_folder, None).find_file_for_has_traits_type(BurstConfiguration) burst_config = BurstConfiguration(project.id) with BurstConfigurationH5( os.path.join(simulator_folder, bc_h5_filename)) as bc_h5: bc_h5.load_into(burst_config) return burst_config @staticmethod def prepare_simulation_name(burst, project_id): simulation_number = dao.get_number_of_bursts(project_id) + 1 if burst.name is None: simulation_name = 'simulation_' + str(simulation_number) else: simulation_name = burst.name return simulation_name, simulation_number def prepare_indexes_for_simulation_results(self, operation, result_filenames, burst): indexes = list() self.logger.debug( "Preparing indexes for simulation results in operation {}...". format(operation.id)) for filename in result_filenames: try: self.logger.debug( "Preparing index for filename: {}".format(filename)) index = h5.index_for_h5_file(filename)() h5_class = h5.REGISTRY.get_h5file_for_index(type(index)) with h5_class(filename) as index_h5: index.fill_from_h5(index_h5) index.fill_from_generic_attributes( index_h5.load_generic_attributes()) index.fk_parent_burst = burst.gid index.fk_from_operation = operation.id if operation.fk_operation_group: datatype_group = dao.get_datatypegroup_by_op_group_id( operation.fk_operation_group) self.logger.debug( "Found DatatypeGroup with id {} for operation {}". format(datatype_group.id, operation.id)) index.fk_datatype_group = datatype_group.id # Update the operation group name operation_group = dao.get_operationgroup_by_id( operation.fk_operation_group) operation_group.fill_operationgroup_name( "TimeSeriesRegionIndex") dao.store_entity(operation_group) self.logger.debug( "Prepared index {} for file {} in operation {}".format( index.summary_info, filename, operation.id)) indexes.append(index) except Exception as e: self.logger.debug( "Skip preparing index {} because there was an error.". format(filename)) self.logger.error(e) self.logger.debug( "Prepared {} indexes for results in operation {}...".format( len(indexes), operation.id)) return indexes def prepare_index_for_metric_result(self, operation, result_filename, burst): self.logger.debug( "Preparing index for metric result in operation {}...".format( operation.id)) index = h5.index_for_h5_file(result_filename)() with DatatypeMeasureH5(result_filename) as dti_h5: index.gid = dti_h5.gid.load().hex index.metrics = json.dumps(dti_h5.metrics.load()) index.fk_source_gid = dti_h5.analyzed_datatype.load().hex index.fk_from_operation = operation.id index.fk_parent_burst = burst.gid datatype_group = dao.get_datatypegroup_by_op_group_id( operation.fk_operation_group) self.logger.debug( "Found DatatypeGroup with id {} for operation {}".format( datatype_group.id, operation.id)) index.fk_datatype_group = datatype_group.id self.logger.debug( "Prepared index {} for results in operation {}...".format( index.summary_info, operation.id)) return index def _update_pse_burst_status(self, burst_config): operations_in_group = dao.get_operations_in_group( burst_config.fk_operation_group) if burst_config.fk_metric_operation_group: operations_in_group.extend( dao.get_operations_in_group( burst_config.fk_metric_operation_group)) operation_statuses = list() for operation in operations_in_group: if not has_finished(operation.status): self.logger.debug( 'Operation {} in group {} is not finished, burst status will not be updated' .format(operation.id, operation.fk_operation_group)) return operation_statuses.append(operation.status) self.logger.debug( 'All operations in burst {} have finished. Will update burst status' .format(burst_config.id)) if STATUS_ERROR in operation_statuses: self.mark_burst_finished( burst_config, BurstConfiguration.BURST_ERROR, 'Some operations in PSE have finished with errors') elif STATUS_CANCELED in operation_statuses: self.mark_burst_finished(burst_config, BurstConfiguration.BURST_CANCELED) else: self.mark_burst_finished(burst_config) def update_burst_status(self, burst_config): if burst_config.fk_operation_group: self._update_pse_burst_status(burst_config) else: operation = dao.get_operation_by_id(burst_config.fk_simulation) message = operation.additional_info if len(message) == 0: message = None self.mark_burst_finished(burst_config, STATUS_FOR_OPERATION[operation.status], message) @staticmethod def prepare_metrics_operation(operation): # TODO reuse from OperationService and do not duplicate logic here parent_burst = dao.get_generic_entity(BurstConfiguration, operation.fk_operation_group, 'fk_operation_group')[0] metric_operation_group_id = parent_burst.fk_metric_operation_group range_values = operation.range_values metric_algo = dao.get_algorithm_by_module(MEASURE_METRICS_MODULE, MEASURE_METRICS_CLASS) metric_operation = Operation(None, operation.fk_launched_by, operation.fk_launched_in, metric_algo.id, status=STATUS_FINISHED, op_group_id=metric_operation_group_id, range_values=range_values) metric_operation.visible = False metric_operation = dao.store_entity(metric_operation) op_dir = FilesHelper().get_project_folder(operation.project, str(metric_operation.id)) return op_dir, metric_operation @staticmethod def handle_range_params_at_loading(burst_config, all_range_parameters): param1, param2 = None, None if burst_config.range1: param1 = RangeParameter.from_json(burst_config.range1) param1.fill_from_default(all_range_parameters[param1.name]) if burst_config.range2 is not None: param2 = RangeParameter.from_json(burst_config.range2) param2.fill_from_default(all_range_parameters[param2.name]) return param1, param2 def prepare_data_for_burst_copy(self, burst_config_id, burst_name_format, project): burst_config = self.load_burst_configuration(burst_config_id) burst_config_copy = burst_config.clone() count = dao.count_bursts_with_name(burst_config.name, burst_config.fk_project) burst_config_copy.name = burst_name_format.format( burst_config.name, count + 1) storage_path = self.files_helper.get_project_folder( project, str(burst_config.fk_simulation)) simulator = h5.load_view_model(burst_config.simulator_gid, storage_path) simulator.generic_attributes = GenericAttributes() return simulator, burst_config_copy @staticmethod def store_burst(burst_config): return dao.store_entity(burst_config) def load_simulation_from_zip(self, zip_file, project): import_service = ImportService() simulator_folder = import_service.import_simulator_configuration_zip( zip_file) simulator_h5_filename = DirLoader( simulator_folder, None).find_file_for_has_traits_type(SimulatorAdapterModel) simulator_h5_filepath = os.path.join(simulator_folder, simulator_h5_filename) simulator = h5.load_view_model_from_file(simulator_h5_filepath) burst_config = self.load_burst_configuration_from_folder( simulator_folder, project) burst_config_copy = burst_config.clone() simulator.generic_attributes.parent_burst = burst_config_copy.gid return simulator, burst_config_copy, simulator_folder
class SimulatorService(object): def __init__(self): self.logger = get_logger(self.__class__.__module__) self.burst_service = BurstService() self.operation_service = OperationService() self.algorithm_service = AlgorithmService() self.files_helper = FilesHelper() @staticmethod def _reset_model(session_stored_simulator): session_stored_simulator.model = type(session_stored_simulator.model)() vi_indexes = session_stored_simulator.determine_indexes_for_chosen_vars_of_interest( ) vi_indexes = numpy.array(list(vi_indexes.values())) for monitor in session_stored_simulator.monitors: monitor.variables_of_interest = vi_indexes def reset_at_connectivity_change(self, is_simulator_copy, form, session_stored_simulator): """ In case the user copies a simulation and changes the Connectivity, we want to reset the Model and Noise parameters because they might not fit to the new Connectivity's nr of regions. """ if is_simulator_copy and form.connectivity.value != session_stored_simulator.connectivity: self._reset_model(session_stored_simulator) if issubclass(type(session_stored_simulator.integrator), IntegratorStochastic): session_stored_simulator.integrator.noise = type( session_stored_simulator.integrator.noise)() def reset_at_surface_change(self, is_simulator_copy, form, session_stored_simulator): """ In case the user copies a surface-simulation and changes the Surface, we want to reset the Model parameters because they might not fit to the new Surface's nr of vertices. """ if is_simulator_copy and ( session_stored_simulator.surface is None and form.surface.value or session_stored_simulator.surface and form.surface.value != session_stored_simulator.surface.surface_gid): self._reset_model(session_stored_simulator) @staticmethod def _set_simulator_range_parameter(simulator, range_parameter_name, range_parameter_value): range_param_name_list = range_parameter_name.split('.') current_attr = simulator for param_name in range_param_name_list[:len(range_param_name_list) - 1]: current_attr = getattr(current_attr, param_name) setattr(current_attr, range_param_name_list[-1], range_parameter_value) def async_launch_and_prepare_simulation(self, burst_config, user, project, simulator_algo, simulator): try: operation = self.operation_service.prepare_operation( user.id, project, simulator_algo, view_model=simulator, burst_gid=burst_config.gid, op_group_id=burst_config.fk_operation_group) burst_config = self.burst_service.update_simulation_fields( burst_config, operation.id, simulator.gid) storage_path = self.files_helper.get_project_folder( project, str(operation.id)) self.burst_service.store_burst_configuration( burst_config, storage_path) wf_errs = 0 try: OperationService().launch_operation(operation.id, True) return operation except Exception as excep: self.logger.error(excep) wf_errs += 1 if burst_config: self.burst_service.mark_burst_finished( burst_config, error_message=str(excep)) self.logger.debug( "Finished launching workflow. The operation was launched successfully, " + str(wf_errs) + " had error on pre-launch steps") except Exception as excep: self.logger.error(excep) if burst_config: self.burst_service.mark_burst_finished( burst_config, error_message=str(excep)) def prepare_simulation_on_server(self, user_id, project, algorithm, zip_folder_path, simulator_file): simulator_vm = h5.load_view_model_from_file(simulator_file) operation = self.operation_service.prepare_operation( user_id, project, algorithm, view_model=simulator_vm) self.async_launch_simulation_on_server(operation, zip_folder_path) return operation def async_launch_simulation_on_server(self, operation, zip_folder_path): try: OperationService().launch_operation(operation.id, True) return operation except Exception as excep: self.logger.error(excep) finally: shutil.rmtree(zip_folder_path) @staticmethod def _set_range_param_in_dict(param_value): if type(param_value) is numpy.ndarray: return param_value[0] elif isinstance(param_value, uuid.UUID): return param_value.hex else: return param_value def async_launch_and_prepare_pse(self, burst_config, user, project, simulator_algo, range_param1, range_param2, session_stored_simulator): try: algo_category = simulator_algo.algorithm_category operation_group = burst_config.operation_group metric_operation_group = burst_config.metric_operation_group operations = [] range_param2_values = [None] if range_param2: range_param2_values = range_param2.get_range_values() first_simulator = None for param1_value in range_param1.get_range_values(): for param2_value in range_param2_values: # Copy, but generate a new GUID for every Simulator in PSE simulator = copy.deepcopy(session_stored_simulator) simulator.gid = uuid.uuid4() self._set_simulator_range_parameter( simulator, range_param1.name, param1_value) ranges = { range_param1.name: self._set_range_param_in_dict(param1_value) } if param2_value is not None: self._set_simulator_range_parameter( simulator, range_param2.name, param2_value) ranges[ range_param2.name] = self._set_range_param_in_dict( param2_value) ranges = json.dumps(ranges) operation = self.operation_service.prepare_operation( user.id, project, simulator_algo, view_model=simulator, ranges=ranges, burst_gid=burst_config.gid, op_group_id=burst_config.fk_operation_group) simulator.range_values = ranges operations.append(operation) if first_simulator is None: first_simulator = simulator first_operation = operations[0] storage_path = self.files_helper.get_project_folder( project, str(first_operation.id)) burst_config = self.burst_service.update_simulation_fields( burst_config, first_operation.id, first_simulator.gid) self.burst_service.store_burst_configuration( burst_config, storage_path) datatype_group = DataTypeGroup( operation_group, operation_id=first_operation.id, fk_parent_burst=burst_config.gid, state=algo_category.defaultdatastate) dao.store_entity(datatype_group) metrics_datatype_group = DataTypeGroup( metric_operation_group, fk_parent_burst=burst_config.gid, state=algo_category.defaultdatastate) dao.store_entity(metrics_datatype_group) wf_errs = 0 for operation in operations: try: OperationService().launch_operation(operation.id, True) except Exception as excep: self.logger.error(excep) wf_errs += 1 self.burst_service.mark_burst_finished( burst_config, error_message=str(excep)) self.logger.debug("Finished launching workflows. " + str(len(operations) - wf_errs) + " were launched successfully, " + str(wf_errs) + " had error on pre-launch steps") return first_operation except Exception as excep: self.logger.error(excep) self.burst_service.mark_burst_finished(burst_config, error_message=str(excep)) @staticmethod def compute_conn_branch_conditions(is_branch, simulator): if not is_branch: return None conn = load.load_entity_by_gid(simulator.connectivity) if conn.number_of_regions: return FilterChain( fields=[FilterChain.datatype + '.number_of_regions'], operations=["=="], values=[conn.number_of_regions]) @staticmethod def validate_first_fragment(form, project_id, conn_idx): conn_count = dao.count_datatypes(project_id, conn_idx) if conn_count == 0: form.connectivity.errors.append( "No connectivity in the project! Simulation cannot be started without " "a connectivity!") def get_simulation_state_index(self, burst_config, simulation_history_class): parent_burst = burst_config.parent_burst_object simulation_state_index = dao.get_generic_entity( simulation_history_class, parent_burst.gid, "fk_parent_burst") if simulation_state_index is None or len(simulation_state_index) < 1: exc = BurstServiceException( "Simulation State not found for %s, thus we are unable to branch from " "it!" % burst_config.name) self.logger.error(exc) raise exc return simulation_state_index