def test_count_datatypes_in_group(self): """ Test that counting dataTypes is correct. Happy flow.""" _, dt_group_id, first_dt, _ = self._create_datatype_group() count = dao.count_datatypes_in_group(dt_group_id) assert count == 2 count = dao.count_datatypes_in_group(first_dt.id) assert count == 0, "There should be no dataType."
def test_count_datatypes_in_group(self): """ Test that counting dataTypes is correct. Happy flow.""" _, dt_group_id, first_dt, _ = self._create_datatype_group() count = dao.count_datatypes_in_group(dt_group_id) self.assertEqual(count, 2) count = dao.count_datatypes_in_group(first_dt.id) self.assertEqual(count, 0, "There should be no dataType.")
def test_count_datatypes_in_group(self, datatype_group_factory): """ Test that counting dataTypes is correct. Happy flow.""" group = datatype_group_factory() count = dao.count_datatypes_in_group(group.id) assert count == 9 datatypes = dao.get_datatypes_from_datatype_group(group.id) count = dao.count_datatypes_in_group(datatypes[0].id) assert count == 0, "There should be no dataType."
def test_tvb_export_for_datatype_group_with_links(self, datatype_group_factory): """ This method checks export of a data type group with Links """ ts_datatype_group, dm_datatype_group = datatype_group_factory( project=self.test_project, store_vm=True, use_time_series_region=True) file_name, file_path, _ = self.export_manager.export_data( ts_datatype_group, self.TVB_LINKED_EXPORTER, self.test_project) assert file_name is not None, "Export process should return a file name" assert file_path is not None, "Export process should return path to export file" assert os.path.exists( file_path), "Could not find export file: %s on disk." % file_path # Now check if the generated file is a correct ZIP file assert zipfile.is_zipfile( file_path), "Generated file is not a valid ZIP file" with closing(zipfile.ZipFile(file_path)) as zip_file: list_of_files = zip_file.namelist() list_of_folders = [] links_folder_found = False for file in list_of_files: dir_name = os.path.dirname(file) if not links_folder_found: if "Links" in dir_name: links_folder_found = True assert file_path is not None, "Export process should return path to export file" if dir_name not in list_of_folders: list_of_folders.append(dir_name) assert links_folder_found is not None, "Links folder was not exported" count_datatypes = dao.count_datatypes_in_group( ts_datatype_group.id) count_datatypes += dao.count_datatypes_in_group( dm_datatype_group.id) # Check if ZIP files contains files for data types and view models (multiple H5 files in case of a Sim) # +1 For Links folder assert count_datatypes + 1 == len(list_of_folders) # +3 for the 3 files in Links folder: Connectivity, Surface, Region Mapping # time series have 6 files, datatype measures have 2 files assert (count_datatypes / 2) * 6 + ( count_datatypes / 2) * 2 + 3 == len(list_of_files)
def test_tvb_export_for_datatype_group(self): """ This method checks export of a data type group """ datatype_group = self.datatypeFactory.create_datatype_group() file_name, file_path, _ = self.export_manager.export_data( datatype_group, self.TVB_EXPORTER, self.project) self.assertTrue(file_name is not None, "Export process should return a file name") self.assertTrue(file_path is not None, "Export process should return path to export file") self.assertTrue(os.path.exists(file_path), "Could not find export file: %s on disk." % file_path) # Now check if the generated file is a correct ZIP file self.assertTrue(zipfile.is_zipfile(file_path), "Generated file is not a valid ZIP file") with closing(zipfile.ZipFile(file_path)) as zip_file: list_of_files = zip_file.namelist() count_datatypes = dao.count_datatypes_in_group(datatype_group.id) # Check if ZIP files contains files for data types + operation self.assertEqual( count_datatypes * 2, len(list_of_files), "Should have 2 x nr datatypes files, one for operations one for datatypes" )
def get_operation_details(self, operation_gid, is_group): """ :returns: an entity OperationOverlayDetails filled with all information for current operation details. """ if is_group: operation_group = self.get_operation_group_by_gid(operation_gid) operation = dao.get_operations_in_group(operation_group.id, False, True) # Reload, to make sure all attributes lazy are populated as well. operation = dao.get_operation_by_gid(operation.gid) no_of_op_in_group = dao.get_operations_in_group(operation_group.id, is_count=True) datatype_group = self.get_datatypegroup_by_op_group_id(operation_group.id) count_result = dao.count_datatypes_in_group(datatype_group.id) else: operation = dao.get_operation_by_gid(operation_gid) if operation is None: return None no_of_op_in_group = 1 count_result = dao.count_resulted_datatypes(operation.id) user_display_name = dao.get_user_by_id(operation.fk_launched_by).display_name burst = dao.get_burst_for_operation_id(operation.id) datatypes_param, all_special_params = self._review_operation_inputs(operation.gid) op_pid = dao.get_operation_process_for_operation(operation.id) op_details = OperationOverlayDetails(operation, user_display_name, len(datatypes_param), count_result, burst, no_of_op_in_group, op_pid) # Add all parameter which are set differently by the user on this Operation. if all_special_params is not None: op_details.add_scientific_fields(all_special_params) return op_details
def test_tvb_export_for_datatype_group(self, datatype_group_factory): """ This method checks export of a data type group """ datatype_group = datatype_group_factory(project=self.test_project, store_vm=True) file_name, file_path, _ = self.export_manager.export_data( datatype_group, self.TVB_EXPORTER, self.test_project) assert file_name is not None, "Export process should return a file name" assert file_path is not None, "Export process should return path to export file" assert os.path.exists( file_path), "Could not find export file: %s on disk." % file_path # Now check if the generated file is a correct ZIP file assert zipfile.is_zipfile( file_path), "Generated file is not a valid ZIP file" with closing(zipfile.ZipFile(file_path)) as zip_file: list_of_files = zip_file.namelist() list_of_folders = [] for file in list_of_files: dir_name = os.path.dirname(file) if dir_name not in list_of_folders: list_of_folders.append(dir_name) count_datatypes = dao.count_datatypes_in_group(datatype_group.id) # Check if ZIP files contains files for data types and view models (multiple H5 files in case of a Sim) assert count_datatypes == len(list_of_folders) assert count_datatypes * 6 == len(list_of_files)
def get_operation_details(self, operation_gid, is_group): """ :returns: an entity OperationOverlayDetails filled with all information for current operation details. """ if is_group: operation_group = self.get_operation_group_by_gid(operation_gid) operation = dao.get_operations_in_group(operation_group.id, False, True) ## Reload, to make sure all attributes lazy are populated as well. operation = dao.get_operation_by_gid(operation.gid) no_of_op_in_group = dao.get_operations_in_group(operation_group.id, is_count=True) datatype_group = self.get_datatypegroup_by_op_group_id(operation_group.id) count_result = dao.count_datatypes_in_group(datatype_group.id) else: operation = dao.get_operation_by_gid(operation_gid) if operation is None: return None no_of_op_in_group = 1 count_result = dao.count_resulted_datatypes(operation.id) username = dao.get_user_by_id(operation.fk_launched_by).username burst = dao.get_burst_for_operation_id(operation.id) datatypes_param, all_special_params = ProjectService._review_operation_inputs(operation.gid) op_pid = dao.get_operation_process_for_operation(operation.id) op_details = OperationOverlayDetails(operation, username, len(datatypes_param), count_result, burst, no_of_op_in_group, op_pid) ## Add all parameter which are set differently by the user on this Operation. if all_special_params is not None: op_details.add_scientific_fields(all_special_params) return op_details
def _update_dt_groups(self, project_id): dt_groups = dao.get_datatypegroup_for_project(project_id) for dt_group in dt_groups: dt_group.count_results = dao.count_datatypes_in_group(dt_group.id) dts_in_group = dao.get_datatypes_from_datatype_group(dt_group.id) if dts_in_group: dt_group.fk_parent_burst = dts_in_group[0].fk_parent_burst dao.store_entity(dt_group)
def mark_burst_finished(self, burst_entity, burst_status=None, error_message=None, store_h5_file=True): """ Mark Burst status field. Also compute 'weight' for current burst: no of operations inside, estimate time on disk... :param burst_entity: BurstConfiguration to be updated, at finish time. :param burst_status: BurstConfiguration status. By default BURST_FINISHED :param error_message: If given, set the status to error and perpetuate the message. """ if burst_status is None: burst_status = BurstConfiguration.BURST_FINISHED if error_message is not None: burst_status = BurstConfiguration.BURST_ERROR try: # If there are any DataType Groups in current Burst, update their counter. burst_dt_groups = dao.get_generic_entity(DataTypeGroup, burst_entity.gid, "fk_parent_burst") for dt_group in burst_dt_groups: dt_group.count_results = dao.count_datatypes_in_group( dt_group.id) dt_group.disk_size, dt_group.subject = dao.get_summary_for_group( dt_group.id) dao.store_entity(dt_group) # Update actual Burst entity fields burst_entity.datatypes_number = dao.count_datatypes_in_burst( burst_entity.gid) burst_entity.status = burst_status burst_entity.error_message = error_message burst_entity.finish_time = datetime.now() dao.store_entity(burst_entity) if store_h5_file: self.store_burst_configuration(burst_entity) except Exception: self.logger.exception( "Could not correctly update Burst status and meta-data!") burst_entity.status = burst_status burst_entity.error_message = "Error when updating Burst Status" burst_entity.finish_time = datetime.now() dao.store_entity(burst_entity) if store_h5_file: self.store_burst_configuration(burst_entity)
def mark_burst_finished(self, burst_entity, error=False, success=False, cancel=False, error_message=None): """ Mark Burst status field. Also compute 'weight' for current burst: no of operations inside, estimate time on disk... :param burst_entity: BurstConfiguration to be updated, at finish time. :param error: When True, burst will be marked as finished with error. :param success: When True, burst will be marked successfully. :param cancel: When True, burst will be marked as user-canceled. """ try: linked_ops_number = dao.get_operations_in_burst(burst_entity.id, is_count=True) linked_datatypes = dao.get_generic_entity(model.DataType, burst_entity.id, "fk_parent_burst") disk_size = linked_ops_number # 1KB for each dataType, considered for operation.xml files dt_group_sizes = dict() for dtype in linked_datatypes: if dtype.disk_size is not None: disk_size = disk_size + dtype.disk_size ### Prepare and compute DataTypeGroup sizes, in case of ranges. if dtype.fk_datatype_group: previous_group_size = dt_group_sizes[dtype.fk_datatype_group] if (dtype.fk_datatype_group in dt_group_sizes) else 0 dt_group_sizes[dtype.fk_datatype_group] = previous_group_size + dtype.disk_size ### If there are any DataType Groups in current Burst, update their counter. burst_dt_groups = dao.get_generic_entity(model.DataTypeGroup, burst_entity.id, "fk_parent_burst") if len(burst_dt_groups) > 0: for dt_group in burst_dt_groups: dt_group.count_results = dao.count_datatypes_in_group(dt_group.id) dt_group.disk_size = dt_group_sizes[dt_group.id] if (dt_group.id in dt_group_sizes) else 0 dao.store_entity(dt_group) ### Update actual Burst entity fields burst_entity.disk_size = disk_size # In KB burst_entity.datatypes_number = len(linked_datatypes) burst_entity.workflows_number = len(dao.get_workflows_for_burst(burst_entity.id)) burst_entity.mark_status(success=success, error=error, cancel=cancel) burst_entity.error_message = error_message dao.store_entity(burst_entity) except Exception, excep: self.logger.error(excep) self.logger.exception("Could not correctly update Burst status and meta-data!") burst_entity.mark_status(error=True) burst_entity.error_message = "Error when updating Burst Status" dao.store_entity(burst_entity)
def test_tvb_export_for_datatype_group(self, datatype_group_factory): """ This method checks export of a data type group """ datatype_group = datatype_group_factory(project=self.test_project) file_name, file_path, _ = self.export_manager.export_data(datatype_group, self.TVB_EXPORTER, self.test_project) assert file_name is not None, "Export process should return a file name" assert file_path is not None, "Export process should return path to export file" assert os.path.exists(file_path), "Could not find export file: %s on disk." % file_path # Now check if the generated file is a correct ZIP file assert zipfile.is_zipfile(file_path), "Generated file is not a valid ZIP file" with closing(zipfile.ZipFile(file_path)) as zip_file: list_of_files = zip_file.namelist() count_datatypes = dao.count_datatypes_in_group(datatype_group.id) # Check if ZIP files contains files for data types assert count_datatypes == len(list_of_files)
def mark_burst_finished(self, burst_entity, burst_status=None, error_message=None): """ Mark Burst status field. Also compute 'weight' for current burst: no of operations inside, estimate time on disk... :param burst_entity: BurstConfiguration to be updated, at finish time. :param burst_status: BurstConfiguration status. By default BURST_FINISHED :param error_message: If given, set the status to error and perpetuate the message. """ if burst_status is None: burst_status = model.BurstConfiguration.BURST_FINISHED if error_message is not None: burst_status = model.BurstConfiguration.BURST_ERROR try: ### If there are any DataType Groups in current Burst, update their counter. burst_dt_groups = dao.get_generic_entity(model.DataTypeGroup, burst_entity.id, "fk_parent_burst") for dt_group in burst_dt_groups: dt_group.count_results = dao.count_datatypes_in_group(dt_group.id) dt_group.disk_size, dt_group.subject = dao.get_summary_for_group(dt_group.id) dao.store_entity(dt_group) ### Update actual Burst entity fields ## 1KB for each dataType, considered for operation.xml files linked_ops_number = dao.get_operations_in_burst(burst_entity.id, is_count=True) burst_entity.disk_size = linked_ops_number + dao.get_disk_size_for_burst(burst_entity.id) # In KB burst_entity.datatypes_number = dao.count_datatypes_in_burst(burst_entity.id) burst_entity.workflows_number = dao.get_workflows_for_burst(burst_entity.id, is_count=True) burst_entity.status = burst_status burst_entity.error_message = error_message burst_entity.finish_time = datetime.now() dao.store_entity(burst_entity) except Exception: self.logger.exception("Could not correctly update Burst status and meta-data!") burst_entity.status = burst_status burst_entity.error_message = "Error when updating Burst Status" burst_entity.finish_time = datetime.now() dao.store_entity(burst_entity)
def test_tvb_export_for_datatype_group(self): """ This method checks export of a data type group """ datatype_group = self.datatypeFactory.create_datatype_group() file_name, file_path, _ = self.export_manager.export_data(datatype_group, self.TVB_EXPORTER, self.project) self.assertTrue(file_name is not None, "Export process should return a file name") self.assertTrue(file_path is not None, "Export process should return path to export file") self.assertTrue(os.path.exists(file_path), "Could not find export file: %s on disk." % file_path) # Now check if the generated file is a correct ZIP file self.assertTrue(zipfile.is_zipfile(file_path), "Generated file is not a valid ZIP file") with closing(zipfile.ZipFile(file_path)) as zip_file: list_of_files = zip_file.namelist() count_datatypes = dao.count_datatypes_in_group(datatype_group.id) # Check if ZIP files contains files for data types + operation self.assertEqual(count_datatypes * 2, len(list_of_files), "Should have 2 x nr datatypes files, one for operations one for datatypes")
def mark_burst_finished(self, burst_entity, error=False, success=False, cancel=False, error_message=None): """ Mark Burst status field. Also compute 'weight' for current burst: no of operations inside, estimate time on disk... :param burst_entity: BurstConfiguration to be updated, at finish time. :param error: When True, burst will be marked as finished with error. :param success: When True, burst will be marked successfully. :param cancel: When True, burst will be marked as user-canceled. """ try: linked_ops_number = dao.get_operations_in_burst(burst_entity.id, is_count=True) linked_datatypes = dao.get_generic_entity(model.DataType, burst_entity.id, "fk_parent_burst") disk_size = linked_ops_number #### 1KB for each dataType, considered for operation.xml files dt_group_sizes = dict() for dtype in linked_datatypes: if dtype.disk_size is not None: disk_size = disk_size + dtype.disk_size ### Prepare and compute DataTypeGroup sizes, in case of ranges. if dtype.fk_datatype_group: previous_group_size = dt_group_sizes[ dtype.fk_datatype_group] if ( dtype.fk_datatype_group in dt_group_sizes) else 0 dt_group_sizes[ dtype. fk_datatype_group] = previous_group_size + dtype.disk_size ### If there are any DataType Groups in current Burst, update their counter. burst_dt_groups = dao.get_generic_entity(model.DataTypeGroup, burst_entity.id, "fk_parent_burst") if len(burst_dt_groups) > 0: for dt_group in burst_dt_groups: dt_group.count_results = dao.count_datatypes_in_group( dt_group.id) dt_group.disk_size = dt_group_sizes[dt_group.id] if ( dt_group.id in dt_group_sizes) else 0 dao.store_entity(dt_group) ### Update actual Burst entity fields burst_entity.disk_size = disk_size ## In KB burst_entity.datatypes_number = len(linked_datatypes) burst_entity.workflows_number = len( dao.get_workflows_for_burst(burst_entity.id)) burst_entity.mark_status(success=success, error=error, cancel=cancel) burst_entity.error_message = error_message dao.store_entity(burst_entity) except Exception, excep: self.logger.error(excep) self.logger.exception( "Could not correctly update Burst status and meta-data!") burst_entity.mark_status(error=True) burst_entity.error_message = "Error when updating Burst Status" dao.store_entity(burst_entity)