def reset_password(self, **data): """ Service Layer for resetting a password. """ if (KEY_USERNAME not in data) or len(data[KEY_USERNAME]) < 1: raise UsernameException("Empty UserName!") if (KEY_EMAIL not in data) or len(data[KEY_EMAIL]) < 1: raise UsernameException("Empty Email!") old_pass, user = None, None try: user_name = data[KEY_USERNAME] email = data[KEY_EMAIL] user = dao.get_user_by_name_email(user_name, email) if user is None: raise UsernameException("Given credentials don't match!") old_pass = user.password new_pass = ''.join(chr(randint(48, 122)) for _ in range(DEFAULT_PASS_LENGTH)) user.password = md5(new_pass).hexdigest() self.edit_user(user, old_pass) self.logger.info("Setting new password for user " + user_name + " !") email_sender.send(FROM_ADDRESS, email, SUBJECT_RECOVERY, TEXT_RECOVERY % (new_pass,)) return TEXT_DISPLAY except Exception, excep: if old_pass and len(old_pass) > 1 and user: user.password = old_pass dao.store_entity(user) self.logger.error("Could not change user password!") self.logger.exception(excep) raise UsernameException(excep.message)
def test_get_users_for_project(self): """ Get all members of a project except the current user. """ user_1 = model.User("test_user1", "test_pass", "*****@*****.**", False, "user") dao.store_entity(user_1) user_2 = model.User("test_user2", "test_pass", "*****@*****.**", False, "user") dao.store_entity(user_2) user_3 = model.User("test_user3", "test_pass", "*****@*****.**", False, "user") dao.store_entity(user_3) user_4 = model.User("test_user4", "test_pass", "*****@*****.**", False, "user") dao.store_entity(user_4) user_5 = model.User("test_user5", "test_pass", "*****@*****.**", False, "user") dao.store_entity(user_5) admin = dao.get_user_by_name("test_user1") member1 = dao.get_user_by_name("test_user2") member2 = dao.get_user_by_name("test_user5") data = dict(name="test_proj", description="test_desc", users=[member1.id, member2.id]) project = ProjectService().store_project(admin, True, None, **data) all_users, members, pag = self.user_service.get_users_for_project(admin.username, project.id) assert len(members) == 2, "More members than there should be." assert len(all_users) == 5, "Admin should not be viewed as member. " \ "Neither should users that were not part of the project's users list." assert pag == 1, "Invalid total pages number." for user in all_users: assert user.username != admin.username, "Admin is in members!"
def test_remove_project_node(self): """ Test removing of a node from a project. """ inserted_project, gid, gid_op = self._create_value_wrapper(self.test_user) project_to_link = model.Project("Link", self.test_user.id, "descript") project_to_link = dao.store_entity(project_to_link) exact_data = dao.get_datatype_by_gid(gid) dao.store_entity(model.Links(exact_data.id, project_to_link.id)) self.assertTrue(dao.get_datatype_by_gid(gid) is not None, "Initialization problem!") operation_id = dao.get_generic_entity(model.Operation, gid_op, 'gid')[0].id op_folder = self.structure_helper.get_project_folder("test_proj", str(operation_id)) self.assertTrue(os.path.exists(op_folder)) sub_files = os.listdir(op_folder) self.assertEqual(2, len(sub_files)) ### Validate that no more files are created than needed. self.project_service._remove_project_node_files(inserted_project.id, gid) sub_files = os.listdir(op_folder) self.assertEqual(1, len(sub_files)) ### operation.xml file should still be there op_folder = self.structure_helper.get_project_folder("Link", str(operation_id + 1)) sub_files = os.listdir(op_folder) self.assertEqual(2, len(sub_files)) self.assertTrue(dao.get_datatype_by_gid(gid) is not None, "Data should still be in DB, because of links") self.project_service._remove_project_node_files(project_to_link.id, gid) self.assertTrue(dao.get_datatype_by_gid(gid) is None) sub_files = os.listdir(op_folder) self.assertEqual(1, len(sub_files))
def create_operation(algorithm=None, test_user=None, test_project=None, operation_status=model.STATUS_FINISHED, parameters="test params"): """ Create persisted operation. :param algorithm: When not None, introspect TVB and TVB_TEST for adapters. :return: Operation entity after persistence. """ if algorithm is None: algorithm = dao.get_algorithm_by_module('tvb.tests.framework.adapters.ndimensionarrayadapter', 'NDimensionArrayAdapter') if test_user is None: test_user = TestFactory.create_user() if test_project is None: test_project = TestFactory.create_project(test_user) meta = {DataTypeMetaData.KEY_SUBJECT: "John Doe", DataTypeMetaData.KEY_STATE: "RAW_DATA"} operation = model.Operation(test_user.id, test_project.id, algorithm.id, parameters, meta=json.dumps(meta), status=operation_status) dao.store_entity(operation) ### Make sure lazy attributes are correctly loaded. return dao.get_operation_by_id(operation.id)
def submit(self, dynamic_gid, dynamic_name): if dao.get_dynamic_by_name(dynamic_name): return {'saved': False, 'msg': 'There is another configuration with the same name'} dynamic = self.get_cached_dynamic(dynamic_gid) model = dynamic.model integrator = dynamic.integrator model_parameters = [] for name in model.ui_configurable_parameters: value = getattr(model, name)[0] model_parameters.append((name, value)) entity = tvb.core.entities.model.Dynamic( dynamic_name, common.get_logged_user().id, model.__class__.__name__, json.dumps(model_parameters), integrator.__class__.__name__, None # todo: serialize integrator parameters # json.dumps(integrator.raw_ui_integrator_parameters) ) dao.store_entity(entity) return {'saved': True}
def reset_password(self, **data): """ Service Layer for resetting a password. """ if (KEY_EMAIL not in data) or len(data[KEY_EMAIL]) < 1: raise UsernameException("Empty Email!") old_pass, user = None, None try: email = data[KEY_EMAIL] name_hint = data[KEY_USERNAME] user = dao.get_user_by_email(email, name_hint) if user is None: raise UsernameException("No singular user could be found for the given data!") old_pass = user.password new_pass = ''.join(chr(randint(48, 122)) for _ in range(DEFAULT_PASS_LENGTH)) user.password = md5(new_pass).hexdigest() self.edit_user(user, old_pass) self.logger.info("Resetting password for email : " + email) email_sender.send(FROM_ADDRESS, email, SUBJECT_RECOVERY, TEXT_RECOVERY % (user.username, new_pass)) return TEXT_DISPLAY except Exception, excep: if old_pass and len(old_pass) > 1 and user: user.password = old_pass dao.store_entity(user) self.logger.exception("Could not change user password!") raise UsernameException(excep.message)
def upgrade_file(self, input_file_name, datatype=None): """ Upgrades the given file to the latest data version. The file will be upgraded sequentially, up until the current version from tvb.basic.config.settings.VersionSettings.DB_STRUCTURE_VERSION :param input_file_name the path to the file which needs to be upgraded :return True, when update was needed and running it was successful. False, the the file is already up to date. """ if self.is_file_up_to_date(input_file_name): # Avoid running the DB update of size, when H5 is not being changed, to speed-up return False file_version = self.get_file_data_version(input_file_name) self.log.info("Updating from version %s , file: %s " % (file_version, input_file_name)) for script_name in self.get_update_scripts(file_version): self.run_update_script(script_name, input_file=input_file_name) if datatype: # Compute and update the disk_size attribute of the DataType in DB: datatype.disk_size = self.files_helper.compute_size_on_disk(input_file_name) dao.store_entity(datatype) return True
def test_validate_user_non_existent(self): """ Flow for trying to validate a user that doesn't exist in the database. """ user = model.User("test_user", "test_pass", "*****@*****.**", True, "user") dao.store_entity(user) assert not self.user_service.validate_user("test_user2"), "Validation done even tho user is non-existent"
def test_remove_project_node(self): """ Test removing of a node from a project. """ inserted_project, gid, gid_op = self._create_value_wrapper(self.test_user) project_to_link = model.Project("Link", self.test_user.id, "descript") project_to_link = dao.store_entity(project_to_link) exact_data = dao.get_datatype_by_gid(gid) dao.store_entity(model.Links(exact_data.id, project_to_link.id)) assert dao.get_datatype_by_gid(gid) is not None, "Initialization problem!" operation_id = dao.get_generic_entity(model.Operation, gid_op, 'gid')[0].id op_folder = self.structure_helper.get_project_folder("test_proj", str(operation_id)) assert os.path.exists(op_folder) sub_files = os.listdir(op_folder) assert 2 == len(sub_files) ### Validate that no more files are created than needed. if(dao.get_system_user() is None): dao.store_entity(model.User(TvbProfile.current.web.admin.SYSTEM_USER_NAME, None, None, True, None)) self.project_service._remove_project_node_files(inserted_project.id, gid) sub_files = os.listdir(op_folder) assert 1 == len(sub_files) ### operation.xml file should still be there op_folder = self.structure_helper.get_project_folder("Link", str(operation_id + 1)) sub_files = os.listdir(op_folder) assert 2 == len(sub_files) assert dao.get_datatype_by_gid(gid) is not None, "Data should still be in DB, because of links" self.project_service._remove_project_node_files(project_to_link.id, gid) assert dao.get_datatype_by_gid(gid) is None sub_files = os.listdir(op_folder) assert 1 == len(sub_files)
def _run_cluster_job(operation_identifier, user_name_label, adapter_instance): """ Threaded Popen It is the function called by the ClusterSchedulerClient in a Thread. This function starts a new process. """ # Load operation so we can estimate the execution time operation = dao.get_operation_by_id(operation_identifier) kwargs = parse_json_parameters(operation.parameters) kwargs = adapter_instance.prepare_ui_inputs(kwargs) time_estimate = int(adapter_instance.get_execution_time_approximation(**kwargs)) hours = int(time_estimate / 3600) minutes = (int(time_estimate) % 3600) / 60 seconds = int(time_estimate) % 60 # Anything lower than 5 hours just use default walltime if hours < 5: walltime = "05:00:00" else: if hours < 10: hours = "0%d" % hours else: hours = str(hours) walltime = "%s:%s:%s" % (hours, str(minutes), str(seconds)) call_arg = TvbProfile.current.cluster.SCHEDULE_COMMAND % (operation_identifier, user_name_label, walltime) LOGGER.info(call_arg) process_ = Popen([call_arg], stdout=PIPE, shell=True) job_id = process_.stdout.read().replace('\n', '').split(TvbProfile.current.cluster.JOB_ID_STRING)[-1] LOGGER.debug("Got jobIdentifier = %s for CLUSTER operationID = %s" % (operation_identifier, job_id)) operation_identifier = model.OperationProcessIdentifier(operation_identifier, job_id=job_id) dao.store_entity(operation_identifier)
def remove_datatype(self, skip_validation=False): """ Called when a Connectivity is to be removed. """ if not skip_validation: associated_ts = dao.get_generic_entity(TimeSeriesRegion, self.handled_datatype.gid, '_connectivity') associated_rm = dao.get_generic_entity(RegionMapping, self.handled_datatype.gid, '_connectivity') associated_stim = dao.get_generic_entity(StimuliRegion, self.handled_datatype.gid, '_connectivity') associated_mes = dao.get_generic_entity(ConnectivityMeasure, self.handled_datatype.gid, '_connectivity') msg = "Connectivity cannot be removed as it is used by at least one " if len(associated_ts) > 0: raise RemoveDataTypeException(msg + " TimeSeriesRegion.") if len(associated_rm) > 0: raise RemoveDataTypeException(msg + " RegionMapping.") if len(associated_stim) > 0: raise RemoveDataTypeException(msg + " StimuliRegion.") if len(associated_mes) > 0: raise RemoveDataTypeException(msg + " ConnectivityMeasure.") #### Update child Connectivities, if any. child_conns = dao.get_generic_entity(Connectivity, self.handled_datatype.gid, '_parent_connectivity') if len(child_conns) > 0: for one_conn in child_conns[1:]: one_conn.parent_connectivity = child_conns[0].gid if child_conns and child_conns[0]: child_conns[0].parent_connectivity = self.handled_datatype.parent_connectivity for one_child in child_conns: dao.store_entity(one_child) ABCRemover.remove_datatype(self, skip_validation)
def _run_cluster_job(operation_identifier, user_name_label, adapter_instance): """ Threaded Popen It is the function called by the ClusterSchedulerClient in a Thread. This function starts a new process. """ # Load operation so we can estimate the execution time operation = dao.get_operation_by_id(operation_identifier) kwargs = parse_json_parameters(operation.parameters) time_estimate = int(adapter_instance.get_execution_time_approximation(**kwargs)) hours = int(time_estimate / 3600) minutes = (int(time_estimate) % 3600) / 60 seconds = int(time_estimate) % 60 # Anything lower than 2 hours just use default walltime if hours < 2: walltime = "02:00:00" elif hours > 23: walltime = "23:59:59" else: walltime = datetime.time(hours, minutes, seconds) walltime = walltime.strftime("%H:%M:%S") call_arg = config.CLUSTER_SCHEDULE_COMMAND % (walltime, operation_identifier, user_name_label) LOGGER.info(call_arg) process_ = Popen([call_arg], stdout=PIPE, shell=True) job_id = process_.stdout.read().replace('\n', '').split('OAR_JOB_ID=')[-1] LOGGER.debug("Got jobIdentifier = %s for CLUSTER operationID = %s" % (operation_identifier, job_id)) operation_identifier = model.OperationProcessIdentifier(operation_identifier, job_id=job_id) dao.store_entity(operation_identifier)
def generate_users(nr_users, nr_projects): """ The generate_users method will create a clean state db with :param nr_users: number of users to be generated (with random roles between CLINICIAN and RESEARCHER and random validated state) :param nr_projects: maximum number of projects to be generated for each user """ users = [] for i in range(nr_users): coin_flip = random.randint(0, 1) role = 'CLINICIAN' if coin_flip == 1 else 'RESEARCHER' password = md5("test").hexdigest() new_user = model.User("gen" + str(i), password, "*****@*****.**", True, role) dao.store_entity(new_user) new_user = dao.get_user_by_name("gen" + str(i)) ExtremeTestFactory.VALIDATION_DICT[new_user.id] = 0 users.append(new_user) for i in range(nr_users): current_user = dao.get_user_by_name("gen" + str(i)) projects_for_user = random.randint(0, nr_projects) for j in range(projects_for_user): data = dict(name='GeneratedProject' + str(i) + '_' + str(j), description='test_desc', users=ExtremeTestFactory.get_users_ids(random.randint(0, nr_users - 3), nr_users, current_user.id, users)) ProjectService().store_project(current_user, True, None, **data) ExtremeTestFactory.VALIDATION_DICT[current_user.id] += 1
def update(): """ Move images previously stored in TVB operation folders, in a single folder/project. """ projects_count = dao.get_all_projects(is_count=True) for page_start in range(0, projects_count, PAGE_SIZE): projects_page = dao.get_all_projects(page_start=page_start, page_end=min(page_start + PAGE_SIZE, projects_count)) for project in projects_page: figures = _figures_in_project(project.id) for figure in figures: figure.file_path = "%s-%s" % (figure.operation.id, figure.file_path) dao.store_entities(figures) project_path = FilesHelper().get_project_folder(project) update_manager = ProjectUpdateManager(project_path) update_manager.run_all_updates() project.version = TvbProfile.current.version.PROJECT_VERSION dao.store_entity(project)
def stop_operation(operation_id): """ Stop a thread for a given operation id """ operation = dao.get_operation_by_id(operation_id) if not operation or operation.status != model.STATUS_STARTED: LOGGER.warning("Operation already stopped or not found is given to stop job: %s" % operation_id) return True LOGGER.debug("Stopping operation: %s" % str(operation_id)) ## Set the thread stop flag to true for thread in CURRENT_ACTIVE_THREADS: if int(thread.operation_id) == operation_id: thread.stop() LOGGER.debug("Found running thread for operation: %d" % operation_id) ## Kill Thread stopped = True operation_process = dao.get_operation_process_for_operation(operation_id) if operation_process is not None: ## Now try to kill the operation if it exists stopped = OperationExecutor.stop_pid(operation_process.pid) if not stopped: LOGGER.debug("Operation %d was probably killed from it's specific thread." % operation_id) else: LOGGER.debug("Stopped OperationExecutor process for %d" % operation_id) ## Mark operation as canceled in DB. operation.mark_cancelled() dao.store_entity(operation) return stopped
def test_launch_operation_HDD_full_space_started_ops(self): """ Test the actual operation flow by executing a test adapter. """ space_taken_by_started = 100 module = "tvb.tests.framework.adapters.testadapter3" class_name = "TestAdapterHDDRequired" group = dao.find_group(module, class_name) started_operation = model.Operation( self.test_user.id, self.test_project.id, group.id, "", status=model.STATUS_STARTED, estimated_disk_size=space_taken_by_started, ) dao.store_entity(started_operation) adapter = FlowService().build_adapter_instance(group) data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = float(adapter.get_required_disk_size(**data) + space_taken_by_started - 1) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.assertRaises( NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user, self.test_project.id, adapter, tmp_folder, **data ) self._assert_no_dt2()
def save_project_to_user(user_id, project_id): """ Mark for current user that the given project is the last one selected. """ user = dao.get_user_by_id(user_id) user.selected_project = project_id dao.store_entity(user)
def remove_datatype(self, skip_validation = False): """ Called when a TimeSeries is removed. """ associated_cv = dao.get_generic_entity(Covariance, self.handled_datatype.gid, '_source') associated_pca = dao.get_generic_entity(PrincipalComponents, self.handled_datatype.gid, '_source') associated_is = dao.get_generic_entity(IndependentComponents, self.handled_datatype.gid, '_source') associated_cc = dao.get_generic_entity(CrossCorrelation, self.handled_datatype.gid, '_source') associated_fr = dao.get_generic_entity(FourierSpectrum, self.handled_datatype.gid, '_source') associated_wv = dao.get_generic_entity(WaveletCoefficients, self.handled_datatype.gid, '_source') associated_cs = dao.get_generic_entity(CoherenceSpectrum, self.handled_datatype.gid, '_source') associated_dm = dao.get_generic_entity(DatatypeMeasure, self.handled_datatype.gid, '_analyzed_datatype') for datatype_measure in associated_dm: datatype_measure._analyed_datatype = None dao.store_entity(datatype_measure) msg = "TimeSeries cannot be removed as it is used by at least one " if not skip_validation: if len(associated_cv) > 0: raise RemoveDataTypeException(msg + " Covariance.") if len(associated_pca) > 0: raise RemoveDataTypeException(msg + " PrincipalComponents.") if len(associated_is) > 0: raise RemoveDataTypeException(msg + " IndependentComponents.") if len(associated_cc) > 0: raise RemoveDataTypeException(msg + " CrossCorrelation.") if len(associated_fr) > 0: raise RemoveDataTypeException(msg + " FourierSpectrum.") if len(associated_wv) > 0: raise RemoveDataTypeException(msg + " WaveletCoefficients.") if len(associated_cs) > 0: raise RemoveDataTypeException(msg + " CoherenceSpectrum.") ABCRemover.remove_datatype(self, skip_validation)
def add_operation_additional_info(self, message): """ Adds additional info on the operation to be displayed in the UI. Usually a warning message. """ current_op = dao.get_operation_by_id(self.operation_id) current_op.additional_info = message dao.store_entity(current_op)
def test_validate_user_validated(self): """ Flow for trying to validate a user that was already validated. """ user = model.User("test_user", "test_pass", "*****@*****.**", True, "user") dao.store_entity(user) assert not self.user_service.validate_user("test_user"), "Validation invalid."
def __upgrade_datatype_list(self, datatypes): """ Upgrade a list of DataTypes to the current version. :param datatypes: The list of DataTypes that should be upgraded. :returns: (nr_of_dts_upgraded_fine, nr_of_dts_upgraded_fault) a two-tuple of integers representing the number of DataTypes for which the upgrade worked fine, and the number of DataTypes for which some kind of fault occurred """ nr_of_dts_upgraded_fine = 0 nr_of_dts_upgraded_fault = 0 for datatype in datatypes: specific_datatype = dao.get_datatype_by_gid(datatype.gid) if isinstance(specific_datatype, MappedType): try: self.upgrade_file(specific_datatype.get_storage_file_path()) nr_of_dts_upgraded_fine += 1 except (MissingDataFileException, FileVersioningException) as ex: # The file is missing for some reason. Just mark the DataType as invalid. datatype.invalid = True dao.store_entity(datatype) nr_of_dts_upgraded_fault += 1 self.log.exception(ex) return nr_of_dts_upgraded_fine, nr_of_dts_upgraded_fault
def test_validate_user_happy_flow(self): """ Standard flow for a validate user action. """ user = model.User("test_user", "test_pass", "*****@*****.**", False, "user") dao.store_entity(user) assert self.user_service.validate_user("test_user"), "Validation failed when it shouldn't have."
def upgrade(migrate_engine): """ Upgrade operations go here. Don't create your own engine; bind migrate_engine to your metadata. """ meta.bind = migrate_engine table = meta.tables['DATA_TYPES_GROUPS'] create_column(COL_RANGES_1, table) create_column(COL_RANGES_2, table) try: ## Iterate DataTypeGroups from previous code-versions and try to update value for the new column. previous_groups = dao.get_generic_entity(model.DataTypeGroup, "0", "no_of_ranges") for group in previous_groups: operation_group = dao.get_operationgroup_by_id(group.fk_operation_group) #group.only_numeric_ranges = operation_group.has_only_numeric_ranges if operation_group.range3 is not None: group.no_of_ranges = 3 elif operation_group.range2 is not None: group.no_of_ranges = 2 elif operation_group.range1 is not None: group.no_of_ranges = 1 else: group.no_of_ranges = 0 dao.store_entity(group) except Exception, excep: ## we can live with a column only having default value. We will not stop the startup. logger = get_logger(__name__) logger.exception(excep)
def clean_database(self, delete_folders=True): """ Deletes data from all tables """ self.cancel_all_operations() LOGGER.warning("Your Database content will be deleted.") try: session = SessionMaker() for table in reversed(model.Base.metadata.sorted_tables): # We don't delete data from some tables, because those are # imported only during introspection which is done one time if table.name not in self.EXCLUDE_TABLES: try: session.open_session() con = session.connection() LOGGER.debug("Executing Delete From Table " + table.name) con.execute(table.delete()) session.commit() except Exception as e: # We cache exception here, in case some table does not exists and # to allow the others to be deleted LOGGER.warning(e) session.rollback() finally: session.close_session() LOGGER.info("Database was cleanup!") except Exception as excep: LOGGER.warning(excep) raise # Now if the database is clean we can delete also project folders on disk if delete_folders: self.delete_project_folders() dao.store_entity(model.User(TvbProfile.current.web.admin.SYSTEM_USER_NAME, None, None, True, None))
def __init__(self): micro_postfix = "_%d" % int(time.time() * 1000000) # Here create all structures needed later for data types creation self.files_helper = FilesHelper() # First create user user = model.User("datatype_factory_user" + micro_postfix, "test_pass", "*****@*****.**" + micro_postfix, True, "user") self.user = dao.store_entity(user) # Now create a project project_service = ProjectService() data = dict(name='DatatypesFactoryProject' + micro_postfix, description='test_desc', users=[]) self.project = project_service.store_project(self.user, True, None, **data) # Create algorithm alg_category = model.AlgorithmCategory('one', True) dao.store_entity(alg_category) ad = model.Algorithm(SIMULATOR_MODULE, SIMULATOR_CLASS, alg_category.id) self.algorithm = dao.get_algorithm_by_module(SIMULATOR_MODULE, SIMULATOR_CLASS) if self.algorithm is None: self.algorithm = dao.store_entity(ad) # Create an operation self.meta = {DataTypeMetaData.KEY_SUBJECT: self.USER_FULL_NAME, DataTypeMetaData.KEY_STATE: self.DATATYPE_STATE} operation = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters', meta=json.dumps(self.meta), status=model.STATUS_FINISHED) self.operation = dao.store_entity(operation)
def update_dt(dt_id, new_create_date): dt = dao.get_datatype_by_id(dt_id) dt.create_date = new_create_date dao.store_entity(dt) # Update MetaData in H5 as well. dt = dao.get_datatype_by_gid(dt.gid) dt.persist_full_metadata()
def create_link(data_ids, project_id): """ For a list of dataType IDs and a project id create all the required links. """ for data in data_ids: link = model.Links(data, project_id) dao.store_entity(link)
def upgrade(migrate_engine): """ Upgrade operations go here. Don't create your own engine; bind migrate_engine to your metadata. """ meta.bind = migrate_engine table = meta.tables['DATA_TYPES_GROUPS'] create_column(COL_RANGES_1, table) create_column(COL_RANGES_2, table) try: ## Iterate DataTypeGroups from previous code-versions and try to update value for the new column. previous_groups = dao.get_generic_entity(model.DataTypeGroup, "0", "no_of_ranges") for group in previous_groups: operation_group = dao.get_operationgroup_by_id(group.fk_operation_group) #group.only_numeric_ranges = operation_group.has_only_numeric_ranges if operation_group.range3 is not None: group.no_of_ranges = 3 elif operation_group.range2 is not None: group.no_of_ranges = 2 elif operation_group.range1 is not None: group.no_of_ranges = 1 else: group.no_of_ranges = 0 dao.store_entity(group) except Exception as excep: ## we can live with a column only having default value. We will not stop the startup. logger = get_logger(__name__) logger.exception(excep) session = SA_SESSIONMAKER() session.execute(text("""UPDATE "OPERATIONS" SET status = CASE WHEN status = 'FINISHED' THEN '4-FINISHED' WHEN status = 'STARTED' THEN '3-STARTED' WHEN status = 'CANCELED' THEN '2-CANCELED' ELSE '1-ERROR' END WHERE status IN ('FINISHED', 'CANCELED', 'STARTED', 'ERROR');""")) session.commit() session.close() try: session = SA_SESSIONMAKER() for sim_state in session.query(SimulationState).filter(SimulationState.fk_datatype_group is not None).all(): session.delete(sim_state) session.commit() session.close() except Exception as excep: ## It might happen that SimulationState table is not yet created, e.g. if user has version 1.0.2 logger = get_logger(__name__) logger.exception(excep)
def __init_algorithmn(self): """ Insert some starting data in the database. """ categ1 = model.AlgorithmCategory('one', True) self.categ1 = dao.store_entity(categ1) ad = model.Algorithm(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS, categ1.id) self.algo_inst = dao.store_entity(ad)
def rename_burst(burst_id, new_name): """ Rename the burst given by burst_id, setting it's new name to burst_name. """ burst = dao.get_burst_by_id(burst_id) burst.name = new_name dao.store_entity(burst)
def test_prepare_name(self): """ Test prepare burst name """ stored_burst = TestFactory.store_burst(self.test_project.id) simulation_tuple = self.burst_service.prepare_simulation_name( stored_burst, self.test_project.id) assert simulation_tuple[0] == 'simulation_' + str(dao.get_number_of_bursts(self.test_project.id) + 1), \ "The default simulation name is not the defined one" busrt_test_name = "Burst Test Name" stored_burst.name = busrt_test_name stored_burst = dao.store_entity(stored_burst) simulation_tuple = self.burst_service.prepare_simulation_name( stored_burst, self.test_project.id) assert simulation_tuple[ 0] == busrt_test_name, "The burst name is not the given one"
def test_adapter_launch(self, connectivity_factory, region_mapping_factory, time_series_region_index_factory): """ Test that the adapters launches and successfully generates a datatype measure entry. """ meta = { DataTypeMetaData.KEY_SUBJECT: "John Doe", DataTypeMetaData.KEY_STATE: "RAW_DATA" } algo = FlowService().get_algorithm_by_module_and_class( IntrospectionRegistry.SIMULATOR_MODULE, IntrospectionRegistry.SIMULATOR_CLASS) self.operation = model_operation.Operation( self.test_user.id, self.test_project.id, algo.id, json.dumps(''), meta=json.dumps(meta), status=model_operation.STATUS_STARTED) self.operation = dao.store_entity(self.operation) # Get connectivity, region_mapping and a dummy time_series_region connectivity = connectivity_factory() region_mapping = region_mapping_factory() dummy_time_series_index = time_series_region_index_factory( connectivity=connectivity, region_mapping=region_mapping) dummy_time_series_index.start_time = 0.0 dummy_time_series_index.sample_period = 1.0 dummy_time_series_index = \ dao.get_generic_entity(dummy_time_series_index.__class__, dummy_time_series_index.gid, 'gid')[0] ts_metric_adapter = TimeseriesMetricsAdapter() form = TimeseriesMetricsAdapterForm() view_model = form.get_view_model()() view_model.time_series = UUID(dummy_time_series_index.gid) form.fill_trait(view_model) ts_metric_adapter.submit_form(form) resulted_metric = ts_metric_adapter.launch(view_model) assert isinstance( resulted_metric, DatatypeMeasureIndex), "Result should be a datatype measure." assert len(resulted_metric.metrics) >= len(list(ts_metric_adapter.get_form().algorithms.choices)), \ "At least a result should have been generated for every metric." for metric_value in json.loads(resulted_metric.metrics).values(): assert isinstance(metric_value, (float, int))
def _import_bursts(project_entity, bursts_dict): """ Re-create old bursts, but keep a mapping between the id it has here and the old-id it had in the project where they were exported, so we can re-add the datatypes to them. """ burst_ids_mapping = {} for old_burst_id in bursts_dict: burst_information = BurstInformation.load_from_dict( bursts_dict[old_burst_id]) burst_entity = model.BurstConfiguration(project_entity.id) burst_entity.from_dict(burst_information.data) burst_entity = dao.store_entity(burst_entity) burst_ids_mapping[int(old_burst_id)] = burst_entity.id # We don't need the data in dictionary form anymore, so update it with new BurstInformation object bursts_dict[old_burst_id] = burst_information return burst_ids_mapping
def prepare_operation(self, user_id, project_id, algorithm, view_model_gid, op_group=None, ranges=None, visible=True): op_group_id = None if op_group: op_group_id = op_group.id if isinstance(view_model_gid, uuid.UUID): view_model_gid = view_model_gid.hex operation = Operation(user_id, project_id, algorithm.id, json.dumps({'gid': view_model_gid}), op_group_id=op_group_id, range_values=ranges) self.logger.debug("Saving Operation(userId=" + str(user_id) + ",projectId=" + str(project_id) + ",algorithmId=" + str(algorithm.id) + ", ops_group= " + str(op_group_id) + ")") operation.visible = visible operation = dao.store_entity(operation) return operation
def persist_operation_state(self, operation, operation_status, message=None): """ Update Operation instance state. Store it in DB and on HDD/ :param operation: Operation instance :param operation_status: new status :param message: message in case of error :return: operation instance changed """ operation.mark_complete(operation_status, message) operation.queue_full = False operation = dao.store_entity(operation) # update burst also burst_config = self.get_burst_for_operation_id(operation.id) if burst_config is not None: burst_status = STATUS_FOR_OPERATION.get(operation_status) self.mark_burst_finished(burst_config, burst_status, message) return operation
def test_read_write_arrays(self): """ Test the filter function when retrieving dataTypes with a filter after a column from a class specific table (e.g. DATA_arraywrapper). """ test_array = numpy.array(range(16)) shapes = [test_array.shape, (2, 8), (2, 2, 4), (2, 2, 2, 2)] storage_path = self.flow_service.file_helper.get_project_folder( self.operation.project, str(self.operation.id)) for i in range(4): datatype_inst = MappedArray(title="dim_" + str(i + 1), d_type="MappedArray", storage_path=storage_path, module="tvb.datatypes.arrays", subject="John Doe", state="RAW", operation_id=self.operation.id) datatype_inst.array_data = test_array.reshape(shapes[i]) result = dao.store_entity(datatype_inst) result.array_data = None inserted_data = self.flow_service.get_available_datatypes( self.test_project.id, "tvb.datatypes.arrays.MappedArray")[0] assert len(inserted_data) == 4, "Found " + str(len(inserted_data)) for i in range(4): ## inserted_data will be retrieved in the opposite order than the insert order actual_datatype = dao.get_generic_entity(MappedArray, inserted_data[3 - i][2], 'gid')[0] assert actual_datatype.length_1d, shapes[i][0] if i > 0: assert actual_datatype.length_2d == shapes[i][1] expected_arr = test_array.reshape(shapes[i]) assert numpy.equal(actual_datatype.array_data, expected_arr).all(),\ str(i + 1) + "D Data not read correctly" actual_datatype.array_data = None ### Check that meta-data are also written for Array attributes. metadata = actual_datatype.get_metadata('array_data') assert actual_datatype.METADATA_ARRAY_MAX in metadata assert metadata[actual_datatype.METADATA_ARRAY_MAX] == 15 assert actual_datatype.METADATA_ARRAY_MIN in metadata assert metadata[actual_datatype.METADATA_ARRAY_MIN] == 0 assert actual_datatype.METADATA_ARRAY_MEAN in metadata assert metadata[actual_datatype.METADATA_ARRAY_MEAN] == 7.5
def create_value_wrapper(test_user, test_project=None): """ Creates a ValueWrapper dataType, and the associated parent Operation. This is also used in ProjectStructureTest. """ if test_project is None: test_project = TestFactory.create_project(test_user, 'test_proj') operation = TestFactory.create_operation(test_user=test_user, test_project=test_project) value_wrapper = ValueWrapper(data_value="5.0", data_name="my_value", data_type="float") op_dir = FilesHelper().get_project_folder(test_project, str(operation.id)) vw_idx = h5.store_complete(value_wrapper, op_dir) vw_idx.fk_from_operation = operation.id vw_idx = dao.store_entity(vw_idx) return test_project, vw_idx.gid, operation
def __populate_project(self, project_path): """ Create and store a Project entity. """ self.logger.debug("Creating project from path: %s" % project_path) project_dict = self.files_helper.read_project_metadata(project_path) project_entity = manager_of_class(model.Project).new_instance() project_entity = project_entity.from_dict(project_dict, self.user_id) try: self.logger.debug("Storing imported project") return dao.store_entity(project_entity) except IntegrityError, excep: self.logger.exception(excep) error_msg = ("Could not import project: %s with gid: %s. There is already a " "project with the same name or gid.") % (project_entity.name, project_entity.gid) raise ProjectImportException(error_msg)
def build(data=None, op=None): ts = time_series_factory(data) if op is None: op = operation_factory() ts_db = TimeSeriesIndex() ts_db.fk_from_operation = op.id ts_db.fill_from_has_traits(ts) ts_h5_path = h5.path_for_stored_index(ts_db) with TimeSeriesH5(ts_h5_path) as f: f.store(ts) f.sample_rate.store(ts.sample_rate) f.nr_dimensions.store(ts.data.ndim) ts_db = dao.store_entity(ts_db) return ts_db
def store_datatype(self, datatype): """This method stores data type into DB""" try: self.logger.debug("Store datatype: %s with Gid: %s" % (datatype.__class__.__name__, datatype.gid)) return dao.store_entity(datatype) except MissingDataSetException: self.logger.error( "Datatype %s has missing data and could not be imported properly." % (datatype, )) os.remove(datatype.get_storage_file_path()) except IntegrityError as excep: self.logger.exception(excep) error_msg = "Could not import data with gid: %s. There is already a one with " \ "the same name or gid." % datatype.gid # Delete file if can't be imported os.remove(datatype.get_storage_file_path()) raise ProjectImportException(error_msg)
def build(row1=None, row2=None, project=None, operation=None, subject=None, state=None): data_type = DummyDataType() data_type.row1 = row1 data_type.row2 = row2 if operation is None: operation = operation_factory(test_project=project) data_type_index = DummyDataTypeIndex(subject=subject, state=state) data_type_index.fk_from_operation = operation.id data_type_index.fill_from_has_traits(data_type) data_type_h5_path = h5.path_for_stored_index(data_type_index) with DummyDataTypeH5(data_type_h5_path) as f: f.store(data_type) data_type_index = dao.store_entity(data_type_index) return data_type_index
def _remove_project_node_files(self, project_id, gid, links, skip_validation=False): """ Delegate removal of a node in the structure of the project. In case of a problem will THROW StructureException. """ try: project = self.find_project(project_id) datatype = dao.get_datatype_by_gid(gid) if links: op = dao.get_operation_by_id(datatype.fk_from_operation) # Instead of deleting, we copy the datatype to the linked project # We also clone the operation new_operation = self.__copy_linked_datatype_before_delete(op, datatype, project, links[0].fk_to_project) # If there is a datatype group and operation group and they were not moved yet to the linked project, # then do it if datatype.fk_datatype_group is not None: dt_group_op = dao.get_operation_by_id(datatype.fk_from_operation) op_group = dao.get_operationgroup_by_id(dt_group_op.fk_operation_group) op_group.fk_launched_in = links[0].fk_to_project dao.store_entity(op_group) burst = dao.get_burst_for_operation_id(op.id) if burst is not None: burst.fk_project = links[0].fk_to_project dao.store_entity(burst) dt_group = dao.get_datatypegroup_by_op_group_id(op_group.id) dt_group.parent_operation = new_operation dt_group.fk_from_operation = new_operation.id dao.store_entity(dt_group) else: # There is no link for this datatype so it has to be deleted specific_remover = get_remover(datatype.type)(datatype) specific_remover.remove_datatype(skip_validation) # Remove burst if dt has one and it still exists if datatype.fk_parent_burst is not None and datatype.is_ts: burst = dao.get_burst_for_operation_id(datatype.fk_from_operation) if burst is not None: dao.remove_entity(BurstConfiguration, burst.id) except RemoveDataTypeException: self.logger.exception("Could not execute operation Node Remove!") raise except FileStructureException: self.logger.exception("Remove operation failed") raise StructureException("Remove operation failed for unknown reasons.Please contact system administrator.")
def __init__(self): now = datetime.now() micro_postfix = "_%d" % now.microsecond # Here create all structures needed later for data types creation self.files_helper = FilesHelper() # First create user user = model.User("datatype_factory_user" + micro_postfix, "test_pass", "*****@*****.**" + micro_postfix, True, "user") self.user = dao.store_entity(user) # Now create a project project_service = ProjectService() data = dict(name='DatatypesFactoryProject' + micro_postfix, description='test_desc', users=[]) self.project = project_service.store_project(self.user, True, None, **data) # Create algorithm alg_category = model.AlgorithmCategory('one', True) dao.store_entity(alg_category) alg_group = model.AlgorithmGroup("test_module1", "classname1", alg_category.id) dao.store_entity(alg_group) algorithm = model.Algorithm(alg_group.id, 'id', name='', req_data='', param_name='', output='') self.algorithm = dao.store_entity(algorithm) #Create an operation self.meta = { DataTypeMetaData.KEY_SUBJECT: self.USER_FULL_NAME, DataTypeMetaData.KEY_STATE: self.DATATYPE_STATE } operation = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters', meta=json.dumps(self.meta), status="FINISHED", method_name=ABCAdapter.LAUNCH_METHOD) self.operation = dao.store_entity(operation)
def build(): time_series_index = time_series_index_factory() time_series = h5.load_from_index(time_series_index) data = numpy.random.random((10, 10, 10, 10, 10)) cross_correlation = temporal_correlations.CrossCorrelation(source=time_series, array_data=data) op = operation_factory() cross_correlation_index = CrossCorrelationIndex() cross_correlation_index.fk_from_operation = op.id cross_correlation_index.fill_from_has_traits(cross_correlation) cross_correlation_h5_path = h5.path_for_stored_index(cross_correlation_index) with CrossCorrelationH5(cross_correlation_h5_path) as f: f.store(cross_correlation) cross_correlation_index = dao.store_entity(cross_correlation_index) return cross_correlation_index
def _prepare_group(self, project_id, existing_dt_group, kwargs): """ Create and store OperationGroup entity, or return None """ # Standard ranges as accepted from UI range1_values = self.get_range_values(kwargs, self._range_name(1)) range2_values = self.get_range_values(kwargs, self._range_name(2)) available_args = self.__expand_arguments([(kwargs, None)], range1_values, self._range_name(1)) available_args = self.__expand_arguments(available_args, range2_values, self._range_name(2)) is_group = False ranges = [] if self._range_name(1) in kwargs and range1_values is not None: is_group = True ranges.append( json.dumps((kwargs[self._range_name(1)], range1_values))) if self._range_name(2) in kwargs and range2_values is not None: is_group = True ranges.append( json.dumps((kwargs[self._range_name(2)], range2_values))) # Now for additional ranges which might be the case for the 'model exploration' last_range_idx = 3 ranger_name = self._range_name(last_range_idx) while ranger_name in kwargs: values_for_range = self.get_range_values(kwargs, ranger_name) available_args = self.__expand_arguments(available_args, values_for_range, ranger_name) last_range_idx += 1 ranger_name = self._range_name(last_range_idx) if last_range_idx > 3: ranges = [ ] # Since we only have 3 fields in db for this just hide it if not is_group: group = None elif existing_dt_group is None: group = OperationGroup(project_id=project_id, ranges=ranges) group = dao.store_entity(group) else: group = existing_dt_group.parent_operation_group return available_args, group
def test_get_inputs_for_op_group(self): """ Tests method get_datatypes_inputs_for_operation_group. The DataType inputs will be from a DataType group. """ project, dt_group_id, first_dt, second_dt = self._create_datatype_group() first_dt.visible = False dao.store_entity(first_dt) second_dt.visible = False dao.store_entity(second_dt) op_group = model.OperationGroup(project.id, "group", "range1[1..2]") op_group = dao.store_entity(op_group) params_1 = json.dumps({"param_5": "1", "param_1": first_dt.gid, "param_6": "2"}) params_2 = json.dumps({"param_5": "1", "param_4": second_dt.gid, "param_6": "5"}) algo_group = dao.find_group('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3') algo = dao.get_algorithm_by_group(algo_group.id) op1 = model.Operation(self.test_user.id, project.id, algo.id, params_1, op_group_id=op_group.id) op2 = model.Operation(self.test_user.id, project.id, algo.id, params_2, op_group_id=op_group.id) dao.store_entities([op1, op2]) inputs = self.project_service.get_datatypes_inputs_for_operation_group(op_group.id, self.relevant_filter) self.assertEqual(len(inputs), 0) inputs = self.project_service.get_datatypes_inputs_for_operation_group(op_group.id, self.full_filter) self.assertEqual(len(inputs), 1, "Incorrect number of dataTypes.") self.assertFalse(first_dt.id == inputs[0].id, "Retrieved wrong dataType.") self.assertFalse(second_dt.id == inputs[0].id, "Retrieved wrong dataType.") self.assertTrue(dt_group_id == inputs[0].id, "Retrieved wrong dataType.") first_dt.visible = True dao.store_entity(first_dt) inputs = self.project_service.get_datatypes_inputs_for_operation_group(op_group.id, self.relevant_filter) self.assertEqual(len(inputs), 1, "Incorrect number of dataTypes.") self.assertFalse(first_dt.id == inputs[0].id, "Retrieved wrong dataType.") self.assertFalse(second_dt.id == inputs[0].id, "Retrieved wrong dataType.") self.assertTrue(dt_group_id == inputs[0].id, "Retrieved wrong dataType.") inputs = self.project_service.get_datatypes_inputs_for_operation_group(op_group.id, self.full_filter) self.assertEqual(len(inputs), 1, "Incorrect number of dataTypes.") self.assertFalse(first_dt.id == inputs[0].id, "Retrieved wrong dataType.") self.assertFalse(second_dt.id == inputs[0].id, "Retrieved wrong dataType.") self.assertTrue(dt_group_id == inputs[0].id, "Retrieved wrong dataType.")
def prepare_operations_for_workflowsteps(self, workflow_step_list, workflows, user_id, burst_id, project_id, group, sim_operations): """ Create and store Operation entities from a list of Workflow Steps. Will be generated workflows x workflow_step_list Operations. For every step in workflow_step_list one OperationGroup and one DataTypeGroup will be created (in case of PSE). """ for step in workflow_step_list: operation_group = None if (group is not None) and not isinstance(step, model.WorkflowStepView): operation_group = model.OperationGroup(project_id=project_id, ranges=group.range_references) operation_group = dao.store_entity(operation_group) operation = None metadata = {DataTypeMetaData.KEY_BURST: burst_id} algo_category = dao.get_algorithm_by_id(step.fk_algorithm) if algo_category is not None: algo_category = algo_category.algorithm_category for wf_idx, workflow in enumerate(workflows): cloned_w_step = step.clone() cloned_w_step.fk_workflow = workflow.id dynamic_params = cloned_w_step.dynamic_param op_params = cloned_w_step.static_param op_params.update(dynamic_params) range_values = None group_id = None if operation_group is not None: group_id = operation_group.id range_values = sim_operations[wf_idx].range_values if not isinstance(step, model.WorkflowStepView): ## For visualization steps, do not create operations, as those are not really needed. metadata, user_group = self._prepare_metadata(metadata, algo_category, operation_group, op_params) operation = model.Operation(user_id, project_id, step.fk_algorithm, json.dumps(op_params, cls=MapAsJson.MapAsJsonEncoder), meta=json.dumps(metadata), op_group_id=group_id, range_values=range_values, user_group=user_group) operation.visible = step.step_visible operation = dao.store_entity(operation) cloned_w_step.fk_operation = operation.id dao.store_entity(cloned_w_step) if operation_group is not None and operation is not None: datatype_group = model.DataTypeGroup(operation_group, operation_id=operation.id, fk_parent_burst=burst_id, state=metadata[DataTypeMetaData.KEY_STATE]) dao.store_entity(datatype_group)
def test_get_inputs_for_operation(self): """ Tests method get_datatype_and_datatypegroup_inputs_for_operation. Verifies filters' influence over results is as expected """ algo_group = dao.find_group('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3') algo = dao.get_algorithm_by_group(algo_group.id) array_wrappers = self._create_mapped_arrays(self.test_project.id) ids = [] for datatype in array_wrappers: ids.append(datatype[0]) datatype = dao.get_datatype_by_id(ids[0]) datatype.visible = False dao.store_entity(datatype) parameters = json.dumps({"param_5": "1", "param_1": array_wrappers[0][2], "param_2": array_wrappers[1][2], "param_3": array_wrappers[2][2], "param_6": "0"}) operation = model.Operation(self.test_user.id, self.test_project.id, algo.id, parameters) operation = dao.store_entity(operation) inputs = self.project_service.get_datatype_and_datatypegroup_inputs_for_operation(operation.gid, self.relevant_filter) self.assertEqual(len(inputs), 2) self.assertTrue(ids[1] in [inputs[0].id, inputs[1].id], "Retrieved wrong dataType.") self.assertTrue(ids[2] in [inputs[0].id, inputs[1].id], "Retrieved wrong dataType.") self.assertFalse(ids[0] in [inputs[0].id, inputs[1].id], "Retrieved wrong dataType.") inputs = self.project_service.get_datatype_and_datatypegroup_inputs_for_operation(operation.gid, self.full_filter) self.assertEqual(len(inputs), 3, "Incorrect number of operations.") self.assertTrue(ids[0] in [inputs[0].id, inputs[1].id, inputs[2].id], "Retrieved wrong dataType.") self.assertTrue(ids[1] in [inputs[0].id, inputs[1].id, inputs[2].id], "Retrieved wrong dataType.") self.assertTrue(ids[2] in [inputs[0].id, inputs[1].id, inputs[2].id], "Retrieved wrong dataType.") project, dt_group_id, first_dt, _ = self._create_datatype_group() first_dt.visible = False dao.store_entity(first_dt) parameters = json.dumps({"other_param": "_", "param_1": first_dt.gid}) operation = model.Operation(self.test_user.id, project.id, algo.id, parameters) operation = dao.store_entity(operation) inputs = self.project_service.get_datatype_and_datatypegroup_inputs_for_operation(operation.gid, self.relevant_filter) self.assertEqual(len(inputs), 0, "Incorrect number of dataTypes.") inputs = self.project_service.get_datatype_and_datatypegroup_inputs_for_operation(operation.gid, self.full_filter) self.assertEqual(len(inputs), 1, "Incorrect number of dataTypes.") self.assertEqual(inputs[0].id, dt_group_id, "Wrong dataType.") self.assertTrue(inputs[0].id != first_dt.id, "Wrong dataType.")
def _populate_algorithm_categories(algo_category): algo_category_instance = dao.filter_category( algo_category.category_name, algo_category.rawinput, algo_category.display, algo_category.launchable, algo_category.order_nr) if algo_category_instance is not None: algo_category_instance.last_introspection_check = datetime.datetime.now( ) algo_category_instance.removed = False else: algo_category_instance = AlgorithmCategory( algo_category.category_name, algo_category.launchable, algo_category.rawinput, algo_category.display, algo_category.defaultdatastate, algo_category.order_nr, datetime.datetime.now()) algo_category_instance = dao.store_entity(algo_category_instance) return algo_category_instance.id
def _populate_algorithms(self, algo_category_class, algo_category_id): for adapter_class in self.introspection_registry.ADAPTERS[ algo_category_class]: try: if not adapter_class.can_be_active(): self.logger.warning( "Skipped Adapter(probably because MATLAB not found):" + str(adapter_class)) stored_adapter = Algorithm( adapter_class.__module__, adapter_class.__name__, algo_category_id, adapter_class.get_group_name(), adapter_class.get_group_description(), adapter_class.get_ui_name(), adapter_class.get_ui_description(), adapter_class.get_ui_subsection(), datetime.datetime.now()) adapter_inst = adapter_class() adapter_form = adapter_inst.get_form() required_datatype = adapter_form.get_required_datatype() if required_datatype is not None: required_datatype = required_datatype.__name__ filters = adapter_form.get_filters() if filters is not None: filters = filters.to_json() stored_adapter.required_datatype = required_datatype stored_adapter.datatype_filter = filters stored_adapter.parameter_name = adapter_form.get_input_name() stored_adapter.outputlist = str(adapter_inst.get_output()) inst_from_db = dao.get_algorithm_by_module( adapter_class.__module__, adapter_class.__name__) if inst_from_db is not None: stored_adapter.id = inst_from_db.id stored_adapter = dao.store_entity(stored_adapter, inst_from_db is not None) adapter_class.stored_adapter = stored_adapter except Exception: self.logger.exception("Could not introspect Adapters file:" + adapter_class.__module__)
def _prepare_metric_operation(self, sim_operation): # type: (Operation) -> Operation metric_algo = dao.get_algorithm_by_module(MEASURE_METRICS_MODULE, MEASURE_METRICS_CLASS) datatype_index = h5.REGISTRY.get_index_for_datatype(TimeSeries) time_series_index = dao.get_generic_entity(datatype_index, sim_operation.id, 'fk_from_operation')[0] view_model = get_class_by_name("{}.{}".format( MEASURE_METRICS_MODULE, MEASURE_METRICS_MODEL_CLASS))() view_model.time_series = time_series_index.gid view_model.algorithms = tuple(choices.values()) range_values = sim_operation.range_values metadata = { DataTypeMetaData.KEY_BURST: time_series_index.fk_parent_burst } metadata, user_group = self._prepare_metadata( metadata, metric_algo.algorithm_category, None, {}) meta_str = json.dumps(metadata) parent_burst = dao.get_generic_entity( BurstConfiguration, time_series_index.fk_parent_burst, 'id')[0] metric_operation_group_id = parent_burst.fk_metric_operation_group metric_operation = Operation(sim_operation.fk_launched_by, sim_operation.fk_launched_in, metric_algo.id, json.dumps({'gid': view_model.gid.hex}), meta_str, op_group_id=metric_operation_group_id, range_values=range_values) metric_operation.visible = False stored_metric_operation = dao.store_entity(metric_operation) metrics_datatype_group = dao.get_generic_entity( DataTypeGroup, metric_operation_group_id, 'fk_operation_group')[0] if metrics_datatype_group.fk_from_operation is None: metrics_datatype_group.fk_from_operation = metric_operation.id self._store_view_model(stored_metric_operation, sim_operation.project, view_model) return stored_metric_operation
def _create_operation(self, project_id, algorithm_id): """ dummy operation :param project_id: the project in which the operation is created :param algorithm_id: the algorithm to be run for the operation :return: a dummy `Operation` with the given specifications """ algorithm = dao.get_algorithm_by_id(algorithm_id) meta = { DataTypeMetaData.KEY_SUBJECT: "John Doe", DataTypeMetaData.KEY_STATE: "RAW_DATA" } operation = Operation(self.test_user.id, project_id, algorithm.id, 'test params', meta=json.dumps(meta), status=STATUS_FINISHED) return dao.store_entity(operation)
def test_get_upload_operations(self): """ Test get_all when filter is for Upload category. """ self.__init_algorithmn() upload_algo = self._create_algo_for_upload() project = model.Project("test_proj_2", self.test_user.id, "desc") project = dao.store_entity(project) op1 = model.Operation(self.test_user.id, self.test_project.id, self.algo_inst.id, "") op2 = model.Operation(self.test_user.id, project.id, upload_algo.id, "", status=model.STATUS_FINISHED) op3 = model.Operation(self.test_user.id, self.test_project.id, upload_algo.id, "") op4 = model.Operation(self.test_user.id, self.test_project.id, upload_algo.id, "", status=model.STATUS_FINISHED) op5 = model.Operation(self.test_user.id, self.test_project.id, upload_algo.id, "", status=model.STATUS_FINISHED) operations = dao.store_entities([op1, op2, op3, op4, op5]) upload_operations = self.project_service.get_all_operations_for_uploaders( self.test_project.id) self.assertEqual(2, len(upload_operations), "Wrong number of upload operations.") upload_ids = [operation.id for operation in upload_operations] for i in [3, 4]: self.assertTrue(operations[i].id in upload_ids, "The operation should be an upload operation.") for i in [0, 1, 2]: self.assertFalse( operations[i].id in upload_ids, "The operation should not be an upload operation.")
def build(connectivity, region_mapping, ts=None, test_user=None, test_project=None, op=None): if ts is None: ts = time_series_region_factory(connectivity, region_mapping) if not op: op = operation_factory(test_user=test_user, test_project=test_project) ts_db = TimeSeriesRegionIndex() ts_db.fk_from_operation = op.id ts_db.fill_from_has_traits(ts) ts_h5_path = h5.path_for_stored_index(ts_db) with TimeSeriesRegionH5(ts_h5_path) as f: f.store(ts) f.sample_rate.store(ts.sample_rate) f.nr_dimensions.store(ts.data.ndim) ts_db = dao.store_entity(ts_db) return ts_db
def build(): time_series_index = time_series_index_factory() time_series = h5.load_from_index(time_series_index) data = numpy.random.random((10, 10)) covariance = graph.Covariance(source=time_series, array_data=data) op = operation_factory() covariance_index = CovarianceIndex() covariance_index.fk_from_operation = op.id covariance_index.fill_from_has_traits(covariance) covariance_h5_path = h5.path_for_stored_index(covariance_index) with CovarianceH5(covariance_h5_path) as f: f.store(covariance) covariance_index = dao.store_entity(covariance_index) return covariance_index
def test_load_burst_history(self): burst_config1 = BurstConfiguration(self.test_project.id) burst_config2 = BurstConfiguration(self.test_project.id) burst_config3 = BurstConfiguration(self.test_project.id) dao.store_entity(burst_config1) dao.store_entity(burst_config2) dao.store_entity(burst_config3) with patch('cherrypy.session', self.sess_mock, create=True): common.add2session(common.KEY_BURST_CONFIG, burst_config1) burst_parameters = self.simulator_controller.load_burst_history() assert len(burst_parameters['burst_list']) == 3, "The burst configurations where not stored."
def test_get_upload_operations(self): """ Test get_all when filter is for Upload category. """ self.__init_algorithmn() upload_algo = self._create_algo_for_upload() project = Project("test_proj_2", self.test_user.id, "desc") project = dao.store_entity(project) op1 = Operation(None, self.test_user.id, self.test_project.id, self.algo_inst.id) op2 = Operation(None, self.test_user.id, project.id, upload_algo.id, status=STATUS_FINISHED) op3 = Operation(None, self.test_user.id, self.test_project.id, upload_algo.id) op4 = Operation(None, self.test_user.id, self.test_project.id, upload_algo.id, status=STATUS_FINISHED) op5 = Operation(None, self.test_user.id, self.test_project.id, upload_algo.id, status=STATUS_FINISHED) operations = dao.store_entities([op1, op2, op3, op4, op5]) upload_operations = self.project_service.get_all_operations_for_uploaders( self.test_project.id) assert 2 == len( upload_operations), "Wrong number of upload operations." upload_ids = [operation.id for operation in upload_operations] for i in [3, 4]: assert operations[i].id in upload_ids, \ "The operation should be an upload operation." for i in [0, 1, 2]: assert not operations[i].id in upload_ids, \ "The operation should not be an upload operation."
def test_get_members_pages(self): """ Create many users (more than one page of members. Create a project and asign all Users as members. Test that 2 pages or Project Members are retrieved. Now remove extra users, to have only one page of members for the project. """ user_ids = [] for i in range(MEMBERS_PAGE_SIZE + 3): user = model_project.User("test_user_no" + str(i), "test_user_no" + str(i), "pass", "*****@*****.**") user = dao.store_entity(user) user_ids.append(user.id) admin = dao.get_user_by_name("test_user_no1") data = dict(name='test_proj', description='test_desc', users=user_ids) project = ProjectService().store_project(admin, True, None, **data) page_users, all_users, pag = self.user_service.get_users_for_project( admin.username, project.id, 2) assert len(page_users) == (MEMBERS_PAGE_SIZE + 3) % MEMBERS_PAGE_SIZE assert len( all_users) == MEMBERS_PAGE_SIZE + 3, 'Not all members returned' assert pag == 2, 'Invalid page number returned' for i in range(3): user = dao.get_user_by_name("test_user_no" + str(i + 2)) self.user_service.delete_user(user.id) page_users, all_users, pag = self.user_service.get_users_for_project( "test_user_no1", project.id, 2) assert len(page_users) == 0, 'Paging not working properly' assert len(all_users) == MEMBERS_PAGE_SIZE, 'Not all members returned' assert pag == 1, 'Invalid page number returned' page_users, all_users, pag = self.user_service.get_users_for_project( "test_user_no1", project.id, 1) assert len( page_users) == MEMBERS_PAGE_SIZE, 'Paging not working properly' assert len(all_users) == MEMBERS_PAGE_SIZE, 'Not all members returned' assert pag == 1, 'Invalid page number returned'
def _prepare_metric_operation(self, sim_operation): # type: (Operation) -> None metric_algo = dao.get_algorithm_by_module( TimeseriesMetricsAdapter.__module__, TimeseriesMetricsAdapter.__name__) time_series_index = dao.get_generic_entity(TimeSeriesIndex, sim_operation.id, 'fk_from_operation')[0] view_model = TimeseriesMetricsAdapterModel() view_model.time_series = time_series_index.gid view_model.algorithms = tuple(choices.values()) range_values = sim_operation.range_values metadata = { DataTypeMetaData.KEY_BURST: time_series_index.fk_parent_burst } metadata, user_group = self._prepare_metadata( metadata, metric_algo.algorithm_category, None, {}) meta_str = json.dumps(metadata) parent_burst = dao.get_generic_entity( BurstConfiguration, time_series_index.fk_parent_burst, 'id')[0] metric_operation_group_id = parent_burst.fk_metric_operation_group metric_operation = Operation(sim_operation.fk_launched_by, sim_operation.fk_launched_in, metric_algo.id, json.dumps({'gid': view_model.gid.hex}), meta_str, op_group_id=metric_operation_group_id, range_values=range_values) metric_operation.visible = False stored_metric_operation = dao.store_entity(metric_operation) metrics_datatype_group = dao.get_generic_entity( DataTypeGroup, metric_operation_group_id, 'fk_operation_group')[0] if metrics_datatype_group.fk_from_operation is None: metrics_datatype_group.fk_from_operation = metric_operation.id OperationService._store_view_model(stored_metric_operation, sim_operation.project, view_model) return stored_metric_operation
def test_get_available_bursts_happy(self): """ Test that all the correct burst are returned for the given project. """ project = Project("second_test_proj", self.test_user.id, "description") second_project = dao.store_entity(project) test_project_bursts = [TestFactory.store_burst(self.test_project.id).id for _ in range(4)] second_project_bursts = [TestFactory.store_burst(second_project.id).id for _ in range(3)] returned_test_project_bursts = [burst.id for burst in self.burst_service.get_available_bursts(self.test_project.id)] returned_second_project_bursts = [burst.id for burst in self.burst_service.get_available_bursts(second_project.id)] assert len(test_project_bursts) == len(returned_test_project_bursts), \ "Incorrect bursts retrieved for project %s." % self.test_project assert len(second_project_bursts) == len(returned_second_project_bursts), \ "Incorrect bursts retrieved for project %s." % second_project assert set(second_project_bursts) == set(returned_second_project_bursts), \ "Incorrect bursts retrieved for project %s." % second_project assert set(test_project_bursts) == set(returned_test_project_bursts), \ "Incorrect bursts retrieved for project %s." % self.test_project