def _export_linked_datatypes(self, project, zip_file): files_helper = FilesHelper() linked_paths = self._get_linked_datatypes_storage_path(project) if not linked_paths: # do not export an empty operation return # Make a import operation which will contain links to other projects alg_group = dao.find_group(TVB_IMPORTER_MODULE, TVB_IMPORTER_CLASS) algo = dao.get_algorithm_by_group(alg_group.id) op = model.Operation(None, project.id, algo.id, '') op.project = project op.algorithm = algo op.id = 'links-to-external-projects' op.start_now() op.mark_complete(model.STATUS_FINISHED) # write operation.xml to disk files_helper.write_operation_metadata(op) op_folder = files_helper.get_operation_folder(op.project.name, op.id) operation_xml = files_helper.get_operation_meta_file_path(op.project.name, op.id) op_folder_name = os.path.basename(op_folder) # add operation.xml zip_file.write(operation_xml, op_folder_name + '/' + os.path.basename(operation_xml)) # add linked datatypes to archive in the import operation for pth in linked_paths: zip_pth = op_folder_name + '/' + os.path.basename(pth) zip_file.write(pth, zip_pth) # remove these files, since we only want them in export archive files_helper.remove_folder(op_folder)
def _export_linked_datatypes(self, project, zip_file): files_helper = FilesHelper() linked_paths = self._get_linked_datatypes_storage_path(project) if not linked_paths: # do not export an empty operation return # Make a import operation which will contain links to other projects algo = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE, TVB_IMPORTER_CLASS) op = model.Operation(None, project.id, algo.id, '') op.project = project op.algorithm = algo op.id = 'links-to-external-projects' op.start_now() op.mark_complete(model.STATUS_FINISHED) # write operation.xml to disk files_helper.write_operation_metadata(op) op_folder = files_helper.get_operation_folder(op.project.name, op.id) operation_xml = files_helper.get_operation_meta_file_path(op.project.name, op.id) op_folder_name = os.path.basename(op_folder) # add operation.xml zip_file.write(operation_xml, op_folder_name + '/' + os.path.basename(operation_xml)) # add linked datatypes to archive in the import operation for pth in linked_paths: zip_pth = op_folder_name + '/' + os.path.basename(pth) zip_file.write(pth, zip_pth) # remove these files, since we only want them in export archive files_helper.remove_folder(op_folder)
def _adapt_epileptor_simulations(): """ Previous Simulations on EpileptorWithPermitivity model, should be converted to use the Epileptor model. As the parameters from the two models are having different ranges and defaults, we do not translate parameters, we only set the Epileptor as model instead of EpileptorPermittivityCoupling, and leave the model params to defaults. """ session = SA_SESSIONMAKER() epileptor_old = "EpileptorPermittivityCoupling" epileptor_new = "Epileptor" param_model = "model" try: all_ep_ops = session.query(model.Operation).filter( model.Operation.parameters.ilike('%"' + epileptor_old + '"%')).all() files_helper = FilesHelper() all_bursts = dict() for ep_op in all_ep_ops: try: op_params = parse_json_parameters(ep_op.parameters) if op_params[param_model] != epileptor_old: LOGGER.debug("Skipping op " + str(op_params[param_model]) + " -- " + str(ep_op)) continue LOGGER.debug("Updating " + str(op_params)) op_params[param_model] = epileptor_new ep_op.parameters = json.dumps(op_params, cls=MapAsJson.MapAsJsonEncoder) LOGGER.debug("New params:" + ep_op.parameters) files_helper.write_operation_metadata(ep_op) burst = dao.get_burst_for_operation_id(ep_op.id) if burst is not None: LOGGER.debug("Updating burst:" + str(burst)) burst.prepare_after_load() burst.simulator_configuration[param_model] = { 'value': epileptor_new } burst._simulator_configuration = json.dumps( burst.simulator_configuration, cls=MapAsJson.MapAsJsonEncoder) if not all_bursts.has_key(burst.id): all_bursts[burst.id] = burst except Exception: LOGGER.exception("Could not process " + str(ep_op)) session.add_all(all_ep_ops) session.add_all(all_bursts.values()) session.commit() except Exception: LOGGER.exception("Could not update Simulation Epileptor Params") finally: session.close()
def _adapt_epileptor_simulations(): """ Previous Simulations on EpileptorWithPermitivity model, should be converted to use the Epileptor model. As the parameters from the two models are having different ranges and defaults, we do not translate parameters, we only set the Epileptor as model instead of EpileptorPermittivityCoupling, and leave the model params to defaults. """ session = SA_SESSIONMAKER() epileptor_old = "EpileptorPermittivityCoupling" epileptor_new = "Epileptor" param_model = "model" try: all_ep_ops = session.query(model.Operation).filter( model.Operation.parameters.ilike('%"' + epileptor_old + '"%')).all() files_helper = FilesHelper() all_bursts = dict() for ep_op in all_ep_ops: try: op_params = parse_json_parameters(ep_op.parameters) if op_params[param_model] != epileptor_old: LOGGER.debug("Skipping op " + str(op_params[param_model]) + " -- " + str(ep_op)) continue LOGGER.debug("Updating " + str(op_params)) op_params[param_model] = epileptor_new ep_op.parameters = json.dumps(op_params, cls=MapAsJson.MapAsJsonEncoder) LOGGER.debug("New params:" + ep_op.parameters) files_helper.write_operation_metadata(ep_op) burst = dao.get_burst_for_operation_id(ep_op.id) if burst is not None: LOGGER.debug("Updating burst:" + str(burst)) burst.prepare_after_load() burst.simulator_configuration[param_model] = {'value': epileptor_new} burst._simulator_configuration = json.dumps(burst.simulator_configuration, cls=MapAsJson.MapAsJsonEncoder) if not all_bursts.has_key(burst.id): all_bursts[burst.id] = burst except Exception: LOGGER.exception("Could not process " + str(ep_op)) session.add_all(all_ep_ops) session.add_all(all_bursts.values()) session.commit() except Exception: LOGGER.exception("Could not update Simulation Epileptor Params") finally: session.close()
class FigureService: """ Service layer for Figure entities. """ _TYPE_PNG = "png" _TYPE_SVG = "svg" _BRANDING_BAR_PNG = os.path.join(os.path.dirname(__file__), "resources", "branding_bar.png") _BRANDING_BAR_SVG = os.path.join(os.path.dirname(__file__), "resources", "branding_bar.svg") _DEFAULT_SESSION_NAME = "Default" _DEFAULT_IMAGE_FILE_NAME = "snapshot." def __init__(self): self.logger = get_logger(self.__class__.__module__) self.file_helper = FilesHelper() def _write_png(self, store_path, export_data): img_data = base64.b64decode(export_data) # decode the image final_image = Image.open( StringIO(img_data)) # place it in a PIL stream branding_bar = Image.open( FigureService._BRANDING_BAR_PNG) # place the branding bar over final_image.paste(branding_bar, (0, final_image.size[1] - branding_bar.size[1]), branding_bar) final_image.save(store_path) # store to disk as PNG def _write_svg(self, store_path, export_data): dom = xml.dom.minidom.parseString(export_data) figureSvg = dom.getElementsByTagName('svg')[ 0] # get the original image dom = xml.dom.minidom.parse(FigureService._BRANDING_BAR_SVG) try: width = float(figureSvg.getAttribute('width').replace('px', '')) height = float(figureSvg.getAttribute('height').replace('px', '')) except ValueError: # defaults when dimensions are not given width = 1024 height = 768 figureSvg.setAttribute("width", str(width)) figureSvg.setAttribute("height", str(height)) finalSvg = dom.createElement('svg') # prepare the final svg brandingSvg = dom.getElementsByTagName('svg')[ 0] # get the branding bar brandingSvg.setAttribute("y", str(height)) # position it below the figure height += float(brandingSvg.getAttribute('height').replace( 'px', '')) # increase original height with branding bar's height finalSvg.setAttribute("width", str(width)) # same width as original figure finalSvg.setAttribute("height", str(height)) finalSvg.appendChild(figureSvg) # add the image finalSvg.appendChild(brandingSvg) # and the branding bar # Generate path where to store image with open(store_path, 'w') as dest: finalSvg.writexml(dest) # store to disk def _image_path(self, project_name, img_type): "Generate path where to store image" images_folder = self.file_helper.get_images_folder(project_name) file_name = FigureService._DEFAULT_IMAGE_FILE_NAME + img_type return utils.get_unique_file_name(images_folder, file_name) @staticmethod def _generate_image_name(project, user, operation, image_name): if not image_name: if operation is not None: # create a name based on the operation that created the image # e.g. TVB-Algo-Name-354 image_name = operation.algorithm.displayname.replace(' ', '-') else: # default to a generic name image_name = "figure" figure_count = dao.get_figure_count(project.id, user.id) + 1 return 'TVB-%s-%s' % (image_name, figure_count) def store_result_figure(self, project, user, img_type, export_data, image_name=None, operation_id=None): """ Store into a file, Result Image and reference in DB. """ store_path, file_name = self._image_path(project.name, img_type) if img_type == FigureService._TYPE_PNG: # PNG file from canvas self._write_png(store_path, export_data) elif img_type == FigureService._TYPE_SVG: # SVG file from svg viewer self._write_svg(store_path, export_data) if operation_id: operation = dao.get_operation_by_id(operation_id) else: operation = None operation_id = None image_name = self._generate_image_name(project, user, operation, image_name) # Store entity into DB entity = model.ResultFigure(operation_id, user.id, project.id, FigureService._DEFAULT_SESSION_NAME, image_name, file_name, img_type) entity = dao.store_entity(entity) # Load instance from DB to have lazy fields loaded figure = dao.load_figure(entity.id) # Write image meta data to disk self.file_helper.write_image_metadata(figure) if operation: # Force writing operation meta data on disk. # This is important later for operation import self.file_helper.write_operation_metadata(operation) def retrieve_result_figures(self, project, user, selected_session_name='all_sessions'): """ Retrieve from DB all the stored Displayer previews that belongs to the specified session. The previews are for current user and project; grouped by session. """ result, previews_info = dao.get_previews(project.id, user.id, selected_session_name) for name in result: for figure in result[name]: figures_folder = self.file_helper.get_images_folder( project.name) figure_full_path = os.path.join(figures_folder, figure.file_path) # Compute the path figure.file_path = utils.path2url_part(figure_full_path) return result, previews_info @staticmethod def load_figure(figure_id): """ Loads a stored figure by its id. """ return dao.load_figure(figure_id) def edit_result_figure(self, figure_id, **data): """ Retrieve and edit a previously stored figure. """ figure = dao.load_figure(figure_id) figure.session_name = data['session_name'] figure.name = data['name'] dao.store_entity(figure) # Load instance from DB to have lazy fields loaded. figure = dao.load_figure(figure_id) # Store figure meta data in an XML attached to the image. self.file_helper.write_image_metadata(figure) def remove_result_figure(self, figure_id): """ Remove figure from DB and file storage. """ figure = dao.load_figure(figure_id) # Delete all figure related files from disk. figures_folder = self.file_helper.get_images_folder( figure.project.name) path2figure = os.path.join(figures_folder, figure.file_path) if os.path.exists(path2figure): os.remove(path2figure) self.file_helper.remove_image_metadata(figure) # Remove figure reference from DB. result = dao.remove_entity(model.ResultFigure, figure_id) return result
class TestFilesHelper(TransactionalTestCase): """ This class contains tests for the tvb.core.entities.file.files_helper module. """ PROJECT_NAME = "test_proj" def transactional_setup_method(self): """ Set up the context needed by the tests. """ self.files_helper = FilesHelper() self.test_user = TestFactory.create_user() self.test_project = TestFactory.create_project(self.test_user, self.PROJECT_NAME) def transactional_teardown_method(self): """ Remove generated project during tests. """ self.delete_project_folders() def test_check_created(self): """ Test standard flows for check created. """ self.files_helper.check_created() assert os.path.exists(root_storage), "Storage not created!" self.files_helper.check_created(os.path.join(root_storage, "test")) assert os.path.exists(root_storage), "Storage not created!" assert os.path.exists(os.path.join(root_storage, "test")), "Test directory not created!" def test_get_project_folder(self): """ Test the get_project_folder method which should create a folder in case it doesn't already exist. """ project_path = self.files_helper.get_project_folder(self.test_project) assert os.path.exists(project_path), "Folder doesn't exist" folder_path = self.files_helper.get_project_folder(self.test_project, "43") assert os.path.exists(project_path), "Folder doesn't exist" assert os.path.exists(folder_path), "Folder doesn't exist" def test_rename_project_structure(self): """ Try to rename the folder structure of a project. Standard flow. """ self.files_helper.get_project_folder(self.test_project) path, name = self.files_helper.rename_project_structure(self.test_project.name, "new_name") assert path != name, "Rename didn't take effect." def test_rename_structure_same_name(self): """ Try to rename the folder structure of a project. Same name. """ self.files_helper.get_project_folder(self.test_project) with pytest.raises(FileStructureException): self.files_helper.rename_project_structure(self.test_project.name, self.PROJECT_NAME) def test_remove_project_structure(self): """ Check that remove project structure deletes the corresponding folder. Standard flow. """ full_path = self.files_helper.get_project_folder(self.test_project) assert os.path.exists(full_path), "Folder was not created." self.files_helper.remove_project_structure(self.test_project.name) assert not os.path.exists(full_path), "Project folder not deleted." def test_write_project_metadata(self): """ Write XML for test-project. """ self.files_helper.write_project_metadata(self.test_project) expected_file = self.files_helper.get_project_meta_file_path(self.PROJECT_NAME) assert os.path.exists(expected_file) project_meta = XMLReader(expected_file).read_metadata() loaded_project = model_project.Project(None, None) loaded_project.from_dict(project_meta, self.test_user.id) assert self.test_project.name == loaded_project.name assert self.test_project.description == loaded_project.description assert self.test_project.gid == loaded_project.gid expected_dict = self.test_project.to_dict()[1] del expected_dict['last_updated'] found_dict = loaded_project.to_dict()[1] del found_dict['last_updated'] self._dictContainsSubset(expected_dict, found_dict) self._dictContainsSubset(found_dict, expected_dict) def test_write_operation_metadata(self): """ Test that a correct XML is created for an operation. """ operation = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project) expected_file = self.files_helper.get_operation_meta_file_path(self.PROJECT_NAME, operation.id) assert not os.path.exists(expected_file) self.files_helper.write_operation_metadata(operation) assert os.path.exists(expected_file) operation_meta = XMLReader(expected_file).read_metadata() loaded_operation = model_operation.Operation(None, None, None, None) loaded_operation.from_dict(operation_meta, dao, user_id=self.test_user.id) expected_dict = operation.to_dict()[1] found_dict = loaded_operation.to_dict()[1] for key, value in expected_dict.items(): assert str(value) == str(found_dict[key]) # Now validate that operation metaData can be also updated assert "new_group_name" != found_dict['user_group'] self.files_helper.update_operation_metadata(self.PROJECT_NAME, "new_group_name", operation.id) found_dict = XMLReader(expected_file).read_metadata() assert "new_group_name" == found_dict['user_group'] def test_remove_dt_happy_flow(self, dummy_datatype_index_factory): """ Happy flow for removing a file related to a DataType. """ datatype = dummy_datatype_index_factory() h5_path = h5.path_for_stored_index(datatype) assert os.path.exists(h5_path), "Test file was not created!" self.files_helper.remove_datatype_file(h5_path) assert not os.path.exists(h5_path), "Test file was not deleted!" def test_remove_dt_non_existent(self, dummy_datatype_index_factory): """ Try to call remove on a dataType with no H5 file. Should work. """ datatype = dummy_datatype_index_factory() h5_path = h5.path_for_stored_index(datatype) wrong_path = os.path.join(h5_path, "WRONG_PATH") assert not os.path.exists(wrong_path) self.files_helper.remove_datatype_file(wrong_path) def test_move_datatype(self, dummy_datatype_index_factory): """ Make sure associated H5 file is moved to a correct new location. """ datatype = dummy_datatype_index_factory(project=self.test_project) old_file_path = h5.path_for_stored_index(datatype) assert os.path.exists(old_file_path), "Test file was not created!" full_path = h5.path_for_stored_index(datatype) self.files_helper.move_datatype(datatype, self.PROJECT_NAME + '2', "1", full_path) assert not os.path.exists(old_file_path), "Test file was not moved!" datatype.fk_from_operation = 43 new_file_path = os.path.join(self.files_helper.get_project_folder(self.PROJECT_NAME + '2', "1"), old_file_path.split("\\")[-1]) assert os.path.exists(new_file_path), "Test file was not created!" def test_find_relative_path(self): """ Tests that relative path is computed properly. """ rel_path = self.files_helper.find_relative_path("/root/up/to/here/test/it/now", "/root/up/to/here") assert rel_path == os.sep.join(["test", "it", "now"]), "Did not extract relative path as expected." def test_remove_files_valid(self): """ Pass a valid list of files and check they are all removed. """ file_list = ["test1", "test2", "test3"] for file_n in file_list: fp = open(file_n, 'w') fp.write('test') fp.close() for file_n in file_list: assert os.path.isfile(file_n) self.files_helper.remove_files(file_list) for file_n in file_list: assert not os.path.isfile(file_n) def test_remove_folder(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" os.mkdir(folder_name) assert os.path.isdir(folder_name), "Folder should be created." self.files_helper.remove_folder(folder_name) assert not os.path.isdir(folder_name), "Folder should be deleted." def test_remove_folder_non_existing_ignore_exc(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" assert not os.path.isdir(folder_name), "Folder should not exist before call." self.files_helper.remove_folder(folder_name, ignore_errors=True) def test_remove_folder_non_existing(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" assert not os.path.isdir(folder_name), "Folder should not exist before call." with pytest.raises(FileStructureException): self.files_helper.remove_folder(folder_name, False) def _dictContainsSubset(self, expected, actual, msg=None): """Checks whether actual is a superset of expected.""" missing = [] mismatched = [] for key, value in expected.items(): if key not in actual: return False elif value != actual[key]: return False return True
class FigureService: """ Service layer for Figure entities. """ _TYPE_PNG = "png" _TYPE_SVG = "svg" _BRANDING_BAR_PNG = os.path.join(os.path.dirname(__file__), "resources", "branding_bar.png") _BRANDING_BAR_SVG = os.path.join(os.path.dirname(__file__), "resources", "branding_bar.svg") _DEFAULT_SESSION_NAME = "Default" _DEFAULT_IMAGE_FILE_NAME = "snapshot." def __init__(self): self.logger = get_logger(self.__class__.__module__) self.file_helper = FilesHelper() def _write_png(self, store_path, export_data): img_data = base64.b64decode(export_data) # decode the image final_image = Image.open(StringIO(img_data)) # place it in a PIL stream branding_bar = Image.open(FigureService._BRANDING_BAR_PNG) # place the branding bar over final_image.paste(branding_bar, (0, final_image.size[1] - branding_bar.size[1]), branding_bar) final_image.save(store_path) # store to disk as PNG def _write_svg(self, store_path, export_data): dom = xml.dom.minidom.parseString(export_data) figureSvg = dom.getElementsByTagName('svg')[0] # get the original image dom = xml.dom.minidom.parse(FigureService._BRANDING_BAR_SVG) try: width = float(figureSvg.getAttribute('width').replace('px', '')) height = float(figureSvg.getAttribute('height').replace('px', '')) except ValueError: # defaults when dimensions are not given width = 1024 height = 768 figureSvg.setAttribute("width", str(width)) figureSvg.setAttribute("height", str(height)) finalSvg = dom.createElement('svg') # prepare the final svg brandingSvg = dom.getElementsByTagName('svg')[0] # get the branding bar brandingSvg.setAttribute("y", str(height)) # position it below the figure height += float(brandingSvg.getAttribute('height').replace('px', '')) # increase original height with branding bar's height finalSvg.setAttribute("width", str(width)) # same width as original figure finalSvg.setAttribute("height", str(height)) finalSvg.appendChild(figureSvg) # add the image finalSvg.appendChild(brandingSvg) # and the branding bar # Generate path where to store image with open(store_path, 'w') as dest: finalSvg.writexml(dest) # store to disk def _image_path(self, project_name, img_type): "Generate path where to store image" images_folder = self.file_helper.get_images_folder(project_name) file_name = FigureService._DEFAULT_IMAGE_FILE_NAME + img_type return utils.get_unique_file_name(images_folder, file_name) @staticmethod def _generate_image_name(project, user, operation, image_name): if not image_name: if operation is not None: # create a name based on the operation that created the image # e.g. TVB-Algo-Name-354 image_name = operation.algorithm.name.replace(' ', '-') else: # default to a generic name image_name = "figure" figure_count = dao.get_figure_count(project.id, user.id) + 1 return 'TVB-%s-%s' % (image_name, figure_count) def store_result_figure(self, project, user, img_type, export_data, image_name=None, operation_id=None): """ Store into a file, Result Image and reference in DB. """ store_path, file_name = self._image_path(project.name, img_type) if img_type == FigureService._TYPE_PNG: # PNG file from canvas self._write_png(store_path, export_data) elif img_type == FigureService._TYPE_SVG: # SVG file from svg viewer self._write_svg(store_path, export_data) if operation_id: operation = dao.get_operation_by_id(operation_id) else: operation = None operation_id = None image_name = self._generate_image_name(project, user, operation, image_name) # Store entity into DB entity = model.ResultFigure(operation_id, user.id, project.id, FigureService._DEFAULT_SESSION_NAME, image_name, file_name, img_type) entity = dao.store_entity(entity) # Load instance from DB to have lazy fields loaded figure = dao.load_figure(entity.id) # Write image meta data to disk self.file_helper.write_image_metadata(figure) if operation: # Force writing operation meta data on disk. # This is important later for operation import self.file_helper.write_operation_metadata(operation) def retrieve_result_figures(self, project, user, selected_session_name='all_sessions'): """ Retrieve from DB all the stored Displayer previews that belongs to the specified session. The previews are for current user and project; grouped by session. """ result, previews_info = dao.get_previews(project.id, user.id, selected_session_name) for name in result: for figure in result[name]: figures_folder = self.file_helper.get_images_folder(project.name) figure_full_path = os.path.join(figures_folder, figure.file_path) # Compute the path figure.file_path = utils.path2url_part(figure_full_path) return result, previews_info @staticmethod def load_figure(figure_id): """ Loads a stored figure by its id. """ return dao.load_figure(figure_id) def edit_result_figure(self, figure_id, **data): """ Retrieve and edit a previously stored figure. """ figure = dao.load_figure(figure_id) figure.session_name = data['session_name'] figure.name = data['name'] dao.store_entity(figure) # Load instance from DB to have lazy fields loaded. figure = dao.load_figure(figure_id) # Store figure meta data in an XML attached to the image. self.file_helper.write_image_metadata(figure) def remove_result_figure(self, figure_id): """ Remove figure from DB and file storage. """ figure = dao.load_figure(figure_id) # Delete all figure related files from disk. figures_folder = self.file_helper.get_images_folder(figure.project.name) path2figure = os.path.join(figures_folder, figure.file_path) if os.path.exists(path2figure): os.remove(path2figure) self.file_helper.remove_image_metadata(figure) # Remove figure reference from DB. result = dao.remove_entity(model.ResultFigure, figure_id) return result
class FigureService: """ Service layer for Figure entities. """ _TYPE_PNG = "png" _TYPE_SVG = "svg" _BRANDING_BAR_PNG = os.path.join(os.path.dirname(__file__), "resources", "branding_bar.png") _BRANDING_BAR_SVG = os.path.join(os.path.dirname(__file__), "resources", "branding_bar.svg") _DEFAULT_SESSION_NAME = "Default" _DEFAULT_IMAGE_FILE_NAME = "snapshot." def __init__(self): self.logger = get_logger(self.__class__.__module__) self.file_helper = FilesHelper() def store_result_figure(self, project, user, img_type, operation_id, export_data): """ Store into a file, Result Image and reference in DB. """ # Generate path where to store image store_path = self.file_helper.get_images_folder(project.name, operation_id) store_path = utils.get_unique_file_name(store_path, FigureService._DEFAULT_IMAGE_FILE_NAME + img_type)[0] file_path = os.path.split(store_path)[1] if img_type == FigureService._TYPE_PNG: # PNG file from canvas imgData = base64.b64decode(export_data) # decode the image fakeImgFile = StringIO(imgData) # PIL.Image only opens from file, so fake one origImg = Image.open(fakeImgFile) brandingBar = Image.open(FigureService._BRANDING_BAR_PNG) finalSize = (origImg.size[0], # original width origImg.size[1] + brandingBar.size[1]) # original height + brandingBar height finalImg = Image.new("RGBA", finalSize) finalImg.paste(origImg, (0, 0)) # add the original image finalImg.paste(brandingBar, (0, origImg.size[1])) # add the branding bar, below the original # the extra width will be discarded finalImg.save(store_path) # store to disk elif img_type == FigureService._TYPE_SVG: # SVG file from svg viewer dom = xml.dom.minidom.parseString(export_data) figureSvg = dom.getElementsByTagName('svg')[0] # get the original image dom = xml.dom.minidom.parse(FigureService._BRANDING_BAR_SVG) brandingSvg = dom.getElementsByTagName('svg')[0] # get the branding bar brandingSvg.setAttribute("y", figureSvg.getAttribute("height")) # position it below the figure finalSvg = dom.createElement('svg') # prepare the final svg width = figureSvg.getAttribute('width').replace('px', '') # same width as original figure finalSvg.setAttribute("width", width) height = float(figureSvg.getAttribute('height').replace('px', '')) # increase original height with height += float(brandingSvg.getAttribute('height').replace('px', '')) # branding bar's height finalSvg.setAttribute("height", str(height)) finalSvg.appendChild(figureSvg) # add the image finalSvg.appendChild(brandingSvg) # and the branding bar # Generate path where to store image dest = open(store_path, 'w') finalSvg.writexml(dest) # store to disk dest.close() operation = dao.get_operation_by_id(operation_id) file_name = 'TVB-%s-%s' % (operation.algorithm.name.replace(' ', '-'), operation_id) # e.g. TVB-Algo-Name-352 # Store entity into DB entity = model.ResultFigure(operation_id, user.id, project.id, FigureService._DEFAULT_SESSION_NAME, file_name, file_path, img_type) entity = dao.store_entity(entity) # Load instance from DB to have lazy fields loaded figure = dao.load_figure(entity.id) # Write image meta data to disk self.file_helper.write_image_metadata(figure) # Force writing operation meta data on disk. # This is important later for operation import self.file_helper.write_operation_metadata(operation) def retrieve_result_figures(self, project, user, selected_session_name='all_sessions'): """ Retrieve from DB all the stored Displayer previews that belongs to the specified session. The previews are for current user and project; grouped by session. """ result, previews_info = dao.get_previews(project.id, user.id, selected_session_name) for name in result: for figure in result[name]: figures_folder = self.file_helper.get_images_folder(project.name, figure.operation.id) figure_full_path = os.path.join(figures_folder, figure.file_path) # Compute the path figure.file_path = utils.path2url_part(figure_full_path) return result, previews_info @staticmethod def load_figure(figure_id): """ Loads a stored figure by its id. """ return dao.load_figure(figure_id) def edit_result_figure(self, figure_id, **data): """ Retrieve and edit a previously stored figure. """ figure = dao.load_figure(figure_id) figure.session_name = data['session_name'] figure.name = data['name'] dao.store_entity(figure) # Load instance from DB to have lazy fields loaded. figure = dao.load_figure(figure_id) # Store figure meta data in an XML attached to the image. self.file_helper.write_image_metadata(figure) def remove_result_figure(self, figure_id): """ Remove figure from DB and file storage. """ figure = dao.load_figure(figure_id) # Delete all figure related files from disk. figures_folder = self.file_helper.get_images_folder(figure.project.name, figure.operation.id) path2figure = os.path.join(figures_folder, figure.file_path) if os.path.exists(path2figure): os.remove(path2figure) self.file_helper.remove_image_metadata(figure) # Remove figure reference from DB. result = dao.remove_entity(model.ResultFigure, figure_id) return result
class ProjectService: """ Services layer for Project entities. """ def __init__(self): self.logger = get_logger(__name__) self.structure_helper = FilesHelper() def store_project(self, current_user, is_create, selected_id, **data): """ We want to create/update a project entity. """ # Validate Unique Name new_name = data["name"] if len(new_name) < 1: raise ProjectServiceException("Invalid project name!") projects_no = dao.count_projects_for_name(new_name, selected_id) if projects_no > 0: err = {'name': 'Please choose another name, this one is used!'} raise formencode.Invalid("Duplicate Name Error", {}, None, error_dict=err) started_operations = dao.get_operation_numbers(selected_id)[1] if started_operations > 0: raise ProjectServiceException("A project can not be renamed while operations are still running!") if is_create: current_proj = Project(new_name, current_user.id, data["description"]) self.structure_helper.get_project_folder(current_proj) else: try: current_proj = dao.get_project_by_id(selected_id) except Exception as excep: self.logger.exception("An error has occurred!") raise ProjectServiceException(str(excep)) if current_proj.name != new_name: self.structure_helper.rename_project_structure(current_proj.name, new_name) current_proj.name = new_name current_proj.description = data["description"] # Commit to make sure we have a valid ID current_proj.refresh_update_date() self.structure_helper.write_project_metadata(current_proj) current_proj = dao.store_entity(current_proj) # Retrieve, to initialize lazy attributes current_proj = dao.get_project_by_id(current_proj.id) # Update share settings on current Project entity visited_pages = [] prj_admin = current_proj.administrator.username if 'visited_pages' in data and data['visited_pages']: visited_pages = data['visited_pages'].split(',') for page in visited_pages: members = UserService.retrieve_users_except(prj_admin, int(page), MEMBERS_PAGE_SIZE)[0] members = [m.id for m in members] dao.delete_members_for_project(current_proj.id, members) selected_user_ids = data["users"] if is_create and current_user.id not in selected_user_ids: # Make the project admin also member of the current project selected_user_ids.append(current_user.id) dao.add_members_to_project(current_proj.id, selected_user_ids) # Finish operation self.logger.debug("Edit/Save OK for project:" + str(current_proj.id) + ' by user:'******'-' result["count"] = one_op[2] result["gid"] = one_op[13] if one_op[3] is not None and one_op[3]: try: operation_group = dao.get_generic_entity(OperationGroup, one_op[3])[0] result["group"] = operation_group.name result["group"] = result["group"].replace("_", " ") result["operation_group_id"] = operation_group.id datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3]) result["datatype_group_gid"] = datatype_group.gid result["gid"] = operation_group.gid ## Filter only viewers for current DataTypeGroup entity: result["view_groups"] = FlowService().get_visualizers_for_group(datatype_group.gid) except Exception: self.logger.exception("We will ignore group on entity:" + str(one_op)) result["datatype_group_gid"] = None else: result['group'] = None result['datatype_group_gid'] = None result["algorithm"] = dao.get_algorithm_by_id(one_op[4]) result["user"] = dao.get_user_by_id(one_op[5]) if type(one_op[6]) is str: result["create"] = string2date(str(one_op[6])) else: result["create"] = one_op[6] if type(one_op[7]) is str: result["start"] = string2date(str(one_op[7])) else: result["start"] = one_op[7] if type(one_op[8]) is str: result["complete"] = string2date(str(one_op[8])) else: result["complete"] = one_op[8] if result["complete"] is not None and result["start"] is not None: result["duration"] = format_timedelta(result["complete"] - result["start"]) result["status"] = one_op[9] result["additional"] = one_op[10] result["visible"] = True if one_op[11] > 0 else False result['operation_tag'] = one_op[12] result['figures'] = None if not result['group']: datatype_results = dao.get_results_for_operation(result['id']) result['results'] = [] for dt in datatype_results: dt_loaded = ABCAdapter.load_entity_by_gid(dt.gid) if dt_loaded: result['results'].append(dt_loaded) else: self.logger.warning("Could not retrieve datatype %s" % str(dt)) operation_figures = dao.get_figures_for_operation(result['id']) # Compute the full path to the figure / image on disk for figure in operation_figures: figures_folder = self.structure_helper.get_images_folder(figure.project.name) figure_full_path = os.path.join(figures_folder, figure.file_path) # Compute the path available from browser figure.figure_path = utils.path2url_part(figure_full_path) result['figures'] = operation_figures else: result['results'] = None operations.append(result) except Exception: ## We got an exception when processing one Operation Row. We will continue with the rest of the rows. self.logger.exception("Could not prepare operation for display:" + str(one_op)) return selected_project, total_ops_nr, operations, pages_no def retrieve_projects_for_user(self, user_id, current_page=1): """ Return a list with all Projects visible for current user. """ start_idx = PROJECTS_PAGE_SIZE * (current_page - 1) total = dao.get_projects_for_user(user_id, is_count=True) available_projects = dao.get_projects_for_user(user_id, start_idx, PROJECTS_PAGE_SIZE) pages_no = total // PROJECTS_PAGE_SIZE + (1 if total % PROJECTS_PAGE_SIZE else 0) for prj in available_projects: fns, sta, err, canceled, pending = dao.get_operation_numbers(prj.id) prj.operations_finished = fns prj.operations_started = sta prj.operations_error = err prj.operations_canceled = canceled prj.operations_pending = pending prj.disk_size = dao.get_project_disk_size(prj.id) prj.disk_size_human = format_bytes_human(prj.disk_size) self.logger.debug("Displaying " + str(len(available_projects)) + " projects in UI for user " + str(user_id)) return available_projects, pages_no @staticmethod def retrieve_all_user_projects(user_id, page_start=0, page_size= PROJECTS_PAGE_SIZE): """ Return a list with all projects visible for current user, without pagination. """ return dao.get_projects_for_user(user_id, page_start=page_start, page_size=page_size) @staticmethod def get_linkable_projects_for_user(user_id, data_id): """ Find projects with are visible for current user, and in which current datatype hasn't been linked yet. """ return dao.get_linkable_projects_for_user(user_id, data_id) @transactional def remove_project(self, project_id): """ Remove Project from DB and File Storage. """ try: project2delete = dao.get_project_by_id(project_id) self.logger.debug("Deleting project: id=" + str(project_id) + ' name=' + project2delete.name) project_bursts = dao.get_bursts_for_project(project_id) for burst in project_bursts: dao.remove_entity(burst.__class__, burst.id) project_datatypes = dao.get_datatypes_in_project(project_id) for one_data in project_datatypes: self.remove_datatype(project_id, one_data.gid, True) links = dao.get_links_for_project(project_id) for one_link in links: dao.remove_entity(Links, one_link.id) self.structure_helper.remove_project_structure(project2delete.name) dao.delete_project(project_id) self.logger.debug("Deleted project: id=" + str(project_id) + ' name=' + project2delete.name) except RemoveDataTypeException as excep: self.logger.exception("Could not execute operation Node Remove!") raise ProjectServiceException(str(excep)) except FileStructureException as excep: self.logger.exception("Could not delete because of rights!") raise ProjectServiceException(str(excep)) except Exception as excep: self.logger.exception(str(excep)) raise ProjectServiceException(str(excep)) # ----------------- Methods for populating Data-Structure Page --------------- @staticmethod def get_datatype_in_group(group): """ Return all dataTypes that are the result of the same DTgroup. """ return dao.get_datatype_in_group(datatype_group_id=group) @staticmethod def get_datatypes_from_datatype_group(datatype_group_id): """ Retrieve all dataType which are part from the given dataType group. """ return dao.get_datatypes_from_datatype_group(datatype_group_id) @staticmethod def load_operation_by_gid(operation_gid): """ Retrieve loaded Operation from DB""" return dao.get_operation_by_gid(operation_gid) @staticmethod def load_operation_lazy_by_gid(operation_gid): """ Retrieve lazy Operation from DB""" return dao.get_operation_lazy_by_gid(operation_gid) @staticmethod def get_operation_group_by_id(operation_group_id): """ Loads OperationGroup from DB""" return dao.get_operationgroup_by_id(operation_group_id) @staticmethod def get_operation_group_by_gid(operation_group_gid): """ Loads OperationGroup from DB""" return dao.get_operationgroup_by_gid(operation_group_gid) @staticmethod def get_operations_in_group(operation_group): """ Return all the operations from an operation group. """ return dao.get_operations_in_group(operation_group.id) @staticmethod def is_upload_operation(operation_gid): """ Returns True only if the operation with the given GID is an upload operation. """ return dao.is_upload_operation(operation_gid) @staticmethod def get_all_operations_for_uploaders(project_id): """ Returns all finished upload operations. """ return dao.get_all_operations_for_uploaders(project_id) def set_operation_and_group_visibility(self, entity_gid, is_visible, is_operation_group=False): """ Sets the operation visibility. If 'is_operation_group' is True than this method will change the visibility for all the operation from the OperationGroup with the GID field equal to 'entity_gid'. """ def set_visibility(op): # workaround: # 'reload' the operation so that it has the project property set. # get_operations_in_group does not eager load it and now we're out of a sqlalchemy session # write_operation_metadata requires that property op = dao.get_operation_by_id(op.id) # end hack op.visible = is_visible self.structure_helper.write_operation_metadata(op) dao.store_entity(op) def set_group_descendants_visibility(operation_group_id): ops_in_group = dao.get_operations_in_group(operation_group_id) for group_op in ops_in_group: set_visibility(group_op) if is_operation_group: op_group_id = dao.get_operationgroup_by_gid(entity_gid).id set_group_descendants_visibility(op_group_id) else: operation = dao.get_operation_by_gid(entity_gid) # we assure that if the operation belongs to a group than the visibility will be changed for the entire group if operation.fk_operation_group is not None: set_group_descendants_visibility(operation.fk_operation_group) else: set_visibility(operation) def get_operation_details(self, operation_gid, is_group): """ :returns: an entity OperationOverlayDetails filled with all information for current operation details. """ if is_group: operation_group = self.get_operation_group_by_gid(operation_gid) operation = dao.get_operations_in_group(operation_group.id, False, True) # Reload, to make sure all attributes lazy are populated as well. operation = dao.get_operation_by_gid(operation.gid) no_of_op_in_group = dao.get_operations_in_group(operation_group.id, is_count=True) datatype_group = self.get_datatypegroup_by_op_group_id(operation_group.id) count_result = dao.count_datatypes_in_group(datatype_group.id) else: operation = dao.get_operation_by_gid(operation_gid) if operation is None: return None no_of_op_in_group = 1 count_result = dao.count_resulted_datatypes(operation.id) user_display_name = dao.get_user_by_id(operation.fk_launched_by).display_name burst = dao.get_burst_for_operation_id(operation.id) datatypes_param, all_special_params = self._review_operation_inputs(operation.gid) op_pid = dao.get_operation_process_for_operation(operation.id) op_details = OperationOverlayDetails(operation, user_display_name, len(datatypes_param), count_result, burst, no_of_op_in_group, op_pid) # Add all parameter which are set differently by the user on this Operation. if all_special_params is not None: op_details.add_scientific_fields(all_special_params) return op_details @staticmethod def get_filterable_meta(): """ Contains all the attributes by which the user can structure the tree of DataTypes """ return DataTypeMetaData.get_filterable_meta() def get_project_structure(self, project, visibility_filter, first_level, second_level, filter_value): """ Find all DataTypes (including the linked ones and the groups) relevant for the current project. In case of a problem, will return an empty list. """ metadata_list = [] dt_list = dao.get_data_in_project(project.id, visibility_filter, filter_value) for dt in dt_list: # Prepare the DT results from DB, for usage in controller, by converting into DataTypeMetaData objects data = {} is_group = False group_op = None dt_entity = dao.get_datatype_by_gid(dt.gid) if dt_entity is None: self.logger.warning("Ignored entity (possibly removed DT class)" + str(dt)) continue # Filter by dt.type, otherwise Links to individual DT inside a group will be mistaken if dt.type == "DataTypeGroup" and dt.parent_operation.operation_group is not None: is_group = True group_op = dt.parent_operation.operation_group # All these fields are necessary here for dynamic Tree levels. data[DataTypeMetaData.KEY_DATATYPE_ID] = dt.id data[DataTypeMetaData.KEY_GID] = dt.gid data[DataTypeMetaData.KEY_NODE_TYPE] = dt.display_type data[DataTypeMetaData.KEY_STATE] = dt.state data[DataTypeMetaData.KEY_SUBJECT] = str(dt.subject) data[DataTypeMetaData.KEY_TITLE] = dt_entity.display_name data[DataTypeMetaData.KEY_RELEVANCY] = dt.visible data[DataTypeMetaData.KEY_LINK] = dt.parent_operation.fk_launched_in != project.id data[DataTypeMetaData.KEY_TAG_1] = dt.user_tag_1 if dt.user_tag_1 else '' data[DataTypeMetaData.KEY_TAG_2] = dt.user_tag_2 if dt.user_tag_2 else '' data[DataTypeMetaData.KEY_TAG_3] = dt.user_tag_3 if dt.user_tag_3 else '' data[DataTypeMetaData.KEY_TAG_4] = dt.user_tag_4 if dt.user_tag_4 else '' data[DataTypeMetaData.KEY_TAG_5] = dt.user_tag_5 if dt.user_tag_5 else '' # Operation related fields: operation_name = CommonDetails.compute_operation_name( dt.parent_operation.algorithm.algorithm_category.displayname, dt.parent_operation.algorithm.displayname) data[DataTypeMetaData.KEY_OPERATION_TYPE] = operation_name data[DataTypeMetaData.KEY_OPERATION_ALGORITHM] = dt.parent_operation.algorithm.displayname data[DataTypeMetaData.KEY_AUTHOR] = dt.parent_operation.user.username data[DataTypeMetaData.KEY_OPERATION_TAG] = group_op.name if is_group else dt.parent_operation.user_group data[DataTypeMetaData.KEY_OP_GROUP_ID] = group_op.id if is_group else None completion_date = dt.parent_operation.completion_date string_year = completion_date.strftime(MONTH_YEAR_FORMAT) if completion_date is not None else "" string_month = completion_date.strftime(DAY_MONTH_YEAR_FORMAT) if completion_date is not None else "" data[DataTypeMetaData.KEY_DATE] = date2string(completion_date) if (completion_date is not None) else '' data[DataTypeMetaData.KEY_CREATE_DATA_MONTH] = string_year data[DataTypeMetaData.KEY_CREATE_DATA_DAY] = string_month data[DataTypeMetaData.KEY_BURST] = dt._parent_burst.name if dt._parent_burst is not None else '-None-' metadata_list.append(DataTypeMetaData(data, dt.invalid)) return StructureNode.metadata2tree(metadata_list, first_level, second_level, project.id, project.name) @staticmethod def get_datatype_details(datatype_gid): """ :returns: an array. First entry in array is an instance of DataTypeOverlayDetails\ The second one contains all the possible states for the specified dataType. """ meta_atts = DataTypeOverlayDetails() states = DataTypeMetaData.STATES try: datatype_result = dao.get_datatype_details(datatype_gid) meta_atts.fill_from_datatype(datatype_result, datatype_result._parent_burst) return meta_atts, states, datatype_result except Exception: ## We ignore exception here (it was logged above, and we want to return no details). return meta_atts, states, None def _remove_project_node_files(self, project_id, gid, skip_validation=False): """ Delegate removal of a node in the structure of the project. In case of a problem will THROW StructureException. """ try: project = self.find_project(project_id) datatype = dao.get_datatype_by_gid(gid) links = dao.get_links_for_datatype(datatype.id) if links: was_link = False for link in links: # This means it's only a link and we need to remove it if link.fk_from_datatype == datatype.id and link.fk_to_project == project.id: dao.remove_entity(Links, link.id) was_link = True if not was_link: # Create a clone of the operation new_op = Operation(dao.get_system_user().id, links[0].fk_to_project, datatype.parent_operation.fk_from_algo, datatype.parent_operation.parameters, datatype.parent_operation.meta_data, datatype.parent_operation.status, datatype.parent_operation.start_date, datatype.parent_operation.completion_date, datatype.parent_operation.fk_operation_group, datatype.parent_operation.additional_info, datatype.parent_operation.user_group, datatype.parent_operation.range_values) new_op = dao.store_entity(new_op) to_project = self.find_project(links[0].fk_to_project).name new_op_loaded = dao.get_operation_by_id(new_op.id) self.structure_helper.write_operation_metadata(new_op_loaded) full_path = h5.path_for_stored_index(datatype) self.structure_helper.move_datatype(datatype, to_project, str(new_op.id), full_path) datatype.fk_from_operation = new_op.id datatype.parent_operation = new_op dao.store_entity(datatype) dao.remove_entity(Links, links[0].id) else: specific_remover = get_remover(datatype.type)(datatype) specific_remover.remove_datatype(skip_validation) h5_path = h5.path_for_stored_index(datatype) self.structure_helper.remove_datatype_file(h5_path) except RemoveDataTypeException: self.logger.exception("Could not execute operation Node Remove!") raise except FileStructureException: self.logger.exception("Remove operation failed") raise StructureException("Remove operation failed for unknown reasons.Please contact system administrator.") def remove_operation(self, operation_id): """ Remove a given operation """ operation = dao.try_get_operation_by_id(operation_id) if operation is not None: self.logger.debug("Deleting operation %s " % operation) datatypes_for_op = dao.get_results_for_operation(operation_id) for dt in reversed(datatypes_for_op): self.remove_datatype(operation.project.id, dt.gid, False) dao.remove_entity(Operation, operation.id) self.structure_helper.remove_operation_data(operation.project.name, operation_id) self.logger.debug("Finished deleting operation %s " % operation) else: self.logger.warning("Attempt to delete operation with id=%s which no longer exists." % operation_id) def remove_datatype(self, project_id, datatype_gid, skip_validation=False): """ Method used for removing a dataType. If the given dataType is a DatatypeGroup or a dataType from a DataTypeGroup than this method will remove the entire group. The operation(s) used for creating the dataType(s) will also be removed. """ datatype = dao.get_datatype_by_gid(datatype_gid) if datatype is None: self.logger.warning("Attempt to delete DT[%s] which no longer exists." % datatype_gid) return is_datatype_group = False if dao.is_datatype_group(datatype_gid): is_datatype_group = True elif datatype.fk_datatype_group is not None: is_datatype_group = True datatype = dao.get_datatype_by_id(datatype.fk_datatype_group) operations_set = [datatype.fk_from_operation] correct = True if is_datatype_group: self.logger.debug("Removing datatype group %s" % datatype) data_list = dao.get_datatypes_from_datatype_group(datatype.id) for adata in data_list: self._remove_project_node_files(project_id, adata.gid, skip_validation) if adata.fk_from_operation not in operations_set: operations_set.append(adata.fk_from_operation) datatype_group = dao.get_datatype_group_by_gid(datatype.gid) dao.remove_datatype(datatype_gid) correct = correct and dao.remove_entity(OperationGroup, datatype_group.fk_operation_group) else: self.logger.debug("Removing datatype %s" % datatype) self._remove_project_node_files(project_id, datatype.gid, skip_validation) ## Remove Operation entity in case no other DataType needs them. project = dao.get_project_by_id(project_id) for operation_id in operations_set: dependent_dt = dao.get_generic_entity(DataType, operation_id, "fk_from_operation") if len(dependent_dt) > 0: ### Do not remove Operation in case DataType still exist referring it. continue correct = correct and dao.remove_entity(Operation, operation_id) ## Make sure Operation folder is removed self.structure_helper.remove_operation_data(project.name, datatype.fk_from_operation) if not correct: raise RemoveDataTypeException("Could not remove DataType " + str(datatype_gid)) def update_metadata(self, submit_data): """ Update DataType/ DataTypeGroup metadata THROW StructureException when input data is invalid. """ new_data = dict() for key in DataTypeOverlayDetails().meta_attributes_list: if key in submit_data: new_data[key] = submit_data[key] if new_data[CommonDetails.CODE_OPERATION_TAG] == '': new_data[CommonDetails.CODE_OPERATION_TAG] = None try: if (CommonDetails.CODE_OPERATION_GROUP_ID in new_data and new_data[CommonDetails.CODE_OPERATION_GROUP_ID] and new_data[CommonDetails.CODE_OPERATION_GROUP_ID] != ''): # We need to edit a group all_data_in_group = dao.get_datatype_in_group(operation_group_id= new_data[CommonDetails.CODE_OPERATION_GROUP_ID]) if len(all_data_in_group) < 1: raise StructureException("Inconsistent group, can not be updated!") datatype_group = dao.get_generic_entity(DataTypeGroup, all_data_in_group[0].fk_datatype_group)[0] all_data_in_group.append(datatype_group) for datatype in all_data_in_group: new_data[CommonDetails.CODE_GID] = datatype.gid self._edit_data(datatype, new_data, True) else: # Get the required DataType and operation from DB to store changes that will be done in XML. gid = new_data[CommonDetails.CODE_GID] datatype = dao.get_datatype_by_gid(gid) self._edit_data(datatype, new_data) except Exception as excep: self.logger.exception(excep) raise StructureException(str(excep)) def _edit_data(self, datatype, new_data, from_group=False): """ Private method, used for editing a meta-data XML file and a DataType row for a given custom DataType entity with new dictionary of data from UI. """ from tvb.basic.traits.types_mapped import MappedType if isinstance(datatype, MappedType) and not os.path.exists(datatype.get_storage_file_path()): if not datatype.invalid: datatype.invalid = True dao.store_entity(datatype) return # 1. First update Operation fields: # Update group field if possible new_group_name = new_data[CommonDetails.CODE_OPERATION_TAG] empty_group_value = (new_group_name is None or new_group_name == "") if from_group: if empty_group_value: raise StructureException("Empty group is not allowed!") group = dao.get_generic_entity(OperationGroup, new_data[CommonDetails.CODE_OPERATION_GROUP_ID]) if group and len(group) > 0 and new_group_name != group[0].name: group = group[0] exists_group = dao.get_generic_entity(OperationGroup, new_group_name, 'name') if exists_group: raise StructureException("Group '" + new_group_name + "' already exists.") group.name = new_group_name dao.store_entity(group) else: operation = dao.get_operation_by_id(datatype.fk_from_operation) operation.user_group = new_group_name dao.store_entity(operation) # 2. Update dateType fields: datatype.subject = new_data[DataTypeOverlayDetails.DATA_SUBJECT] datatype.state = new_data[DataTypeOverlayDetails.DATA_STATE] if DataTypeOverlayDetails.DATA_TAG_1 in new_data: datatype.user_tag_1 = new_data[DataTypeOverlayDetails.DATA_TAG_1] if DataTypeOverlayDetails.DATA_TAG_2 in new_data: datatype.user_tag_2 = new_data[DataTypeOverlayDetails.DATA_TAG_2] if DataTypeOverlayDetails.DATA_TAG_3 in new_data: datatype.user_tag_3 = new_data[DataTypeOverlayDetails.DATA_TAG_3] if DataTypeOverlayDetails.DATA_TAG_4 in new_data: datatype.user_tag_4 = new_data[DataTypeOverlayDetails.DATA_TAG_4] if DataTypeOverlayDetails.DATA_TAG_5 in new_data: datatype.user_tag_5 = new_data[DataTypeOverlayDetails.DATA_TAG_5] datatype = dao.store_entity(datatype) # 3. Update MetaData in H5 as well. datatype.persist_full_metadata() # 4. Update the group_name/user_group into the operation meta-data file operation = dao.get_operation_by_id(datatype.fk_from_operation) self.structure_helper.update_operation_metadata(operation.project.name, new_group_name, str(datatype.fk_from_operation), from_group) def get_datatype_and_datatypegroup_inputs_for_operation(self, operation_gid, selected_filter): """ Returns the dataTypes that are used as input parameters for the given operation. 'selected_filter' - is expected to be a visibility filter. If any dataType is part of a dataType group then the dataType group will be returned instead of that dataType. """ all_datatypes = self._review_operation_inputs(operation_gid)[0] datatype_inputs = [] for datatype in all_datatypes: if selected_filter.display_name == StaticFiltersFactory.RELEVANT_VIEW: if datatype.visible: datatype_inputs.append(datatype) else: datatype_inputs.append(datatype) datatypes = [] datatype_groups = dict() for data_type in datatype_inputs: if data_type.fk_datatype_group is None: datatypes.append(data_type) elif data_type.fk_datatype_group not in datatype_groups: dt_group = dao.get_datatype_by_id(data_type.fk_datatype_group) datatype_groups[data_type.fk_datatype_group] = dt_group datatypes.extend([v for _, v in six.iteritems(datatype_groups)]) return datatypes def _review_operation_inputs(self, operation_gid): """ :returns: A list of DataTypes that are used as input parameters for the specified operation. And a dictionary will all operation parameters different then the default ones. """ operation = dao.get_operation_by_gid(operation_gid) parameters = json.loads(operation.parameters) try: adapter = ABCAdapter.build_adapter(operation.algorithm) return adapter.review_operation_inputs(parameters) except Exception: self.logger.exception("Could not load details for operation %s" % operation_gid) inputs_datatypes = [] changed_parameters = dict(Warning="Algorithm changed dramatically. We can not offer more details") for submit_param in parameters.values(): self.logger.debug("Searching DT by GID %s" % submit_param) datatype = ABCAdapter.load_entity_by_gid(str(submit_param)) if datatype is not None: inputs_datatypes.append(datatype) return inputs_datatypes, changed_parameters def get_datatypes_inputs_for_operation_group(self, group_id, selected_filter): """ Returns the dataType inputs for an operation group. If more dataTypes are part of the same dataType group then only the dataType group will be returned instead of them. """ operations_gids = dao.get_operations_in_group(group_id, only_gids=True) op_group_inputs = dict() for gid in operations_gids: op_inputs = self.get_datatype_and_datatypegroup_inputs_for_operation(gid[0], selected_filter) for datatype in op_inputs: op_group_inputs[datatype.id] = datatype return list(op_group_inputs.values()) @staticmethod def get_results_for_operation(operation_id, selected_filter=None): """ Retrieve the DataTypes entities resulted after the execution of the given operation. """ return dao.get_results_for_operation(operation_id, selected_filter) @staticmethod def get_operations_for_datatype_group(datatype_group_id, visibility_filter, only_in_groups=False): """ Returns all the operations which uses as an input parameter a dataType from the given DataTypeGroup. visibility_filter - is a filter used for retrieving all the operations or only the relevant ones. If only_in_groups is True than this method will return only the operations that are part from an operation group, otherwise it will return only the operations that are NOT part of an operation group. """ if visibility_filter.display_name != StaticFiltersFactory.RELEVANT_VIEW: return dao.get_operations_for_datatype_group(datatype_group_id, only_relevant=False, only_in_groups=only_in_groups) return dao.get_operations_for_datatype_group(datatype_group_id, only_in_groups=only_in_groups) @staticmethod def get_operations_for_datatype(datatype_gid, visibility_filter, only_in_groups=False): """ Returns all the operations which uses as an input parameter the dataType with the specified GID. If only_in_groups is True than this method will return only the operations that are part from an operation group, otherwise it will return only the operations that are NOT part of an operation group. """ if visibility_filter.display_name != StaticFiltersFactory.RELEVANT_VIEW: return dao.get_operations_for_datatype(datatype_gid, only_relevant=False, only_in_groups=only_in_groups) return dao.get_operations_for_datatype(datatype_gid, only_in_groups=only_in_groups) @staticmethod def get_datatype_by_id(datatype_id): """Retrieve a DataType DB reference by its id.""" return dao.get_datatype_by_id(datatype_id) @staticmethod def get_datatypegroup_by_gid(datatypegroup_gid): """ Returns the DataTypeGroup with the specified gid. """ return dao.get_datatype_group_by_gid(datatypegroup_gid) @staticmethod def count_datatypes_generated_from(datatype_gid): """ A list with all the datatypes resulted from operations that had as input the datatype given by 'datatype_gid'. """ return dao.count_datatypes_generated_from(datatype_gid) @staticmethod def get_datatypegroup_by_op_group_id(operation_group_id): """ Returns the DataTypeGroup with the specified id. """ return dao.get_datatypegroup_by_op_group_id(operation_group_id) @staticmethod def get_datatypes_in_project(project_id, only_visible=False): return dao.get_data_in_project(project_id, only_visible) @staticmethod def set_datatype_visibility(datatype_gid, is_visible): """ Sets the dataType visibility. If the given dataType is a dataType group or it is part of a dataType group than this method will set the visibility for each dataType from this group. """ def set_visibility(dt): """ set visibility flag, persist in db and h5""" dt.visible = is_visible dt = dao.store_entity(dt) dt.persist_full_metadata() def set_group_descendants_visibility(datatype_group_id): datatypes_in_group = dao.get_datatypes_from_datatype_group(datatype_group_id) for group_dt in datatypes_in_group: set_visibility(group_dt) datatype = dao.get_datatype_by_gid(datatype_gid) if isinstance(datatype, DataTypeGroup): # datatype is a group set_group_descendants_visibility(datatype.id) elif datatype.fk_datatype_group is not None: # datatype is member of a group set_group_descendants_visibility(datatype.fk_datatype_group) # the datatype to be updated is the parent datatype group datatype = dao.get_datatype_by_id(datatype.fk_datatype_group) # update the datatype or datatype group. set_visibility(datatype) @staticmethod def is_datatype_group(datatype_gid): """ Used to check if the dataType with the specified GID is a DataTypeGroup. """ return dao.is_datatype_group(datatype_gid)
class BurstService(object): def __init__(self): self.logger = get_logger(self.__class__.__module__) self.file_helper = FilesHelper() def mark_burst_finished(self, burst_entity, burst_status=None, error_message=None): """ Mark Burst status field. Also compute 'weight' for current burst: no of operations inside, estimate time on disk... :param burst_entity: BurstConfiguration to be updated, at finish time. :param burst_status: BurstConfiguration status. By default BURST_FINISHED :param error_message: If given, set the status to error and perpetuate the message. """ if burst_status is None: burst_status = BurstConfiguration.BURST_FINISHED if error_message is not None: burst_status = BurstConfiguration.BURST_ERROR try: # If there are any DataType Groups in current Burst, update their counter. burst_dt_groups = dao.get_generic_entity(DataTypeGroup, burst_entity.id, "fk_parent_burst") for dt_group in burst_dt_groups: dt_group.count_results = dao.count_datatypes_in_group( dt_group.id) dt_group.disk_size, dt_group.subject = dao.get_summary_for_group( dt_group.id) dao.store_entity(dt_group) # Update actual Burst entity fields burst_entity.datatypes_number = dao.count_datatypes_in_burst( burst_entity.id) burst_entity.status = burst_status burst_entity.error_message = error_message burst_entity.finish_time = datetime.now() dao.store_entity(burst_entity) except Exception: self.logger.exception( "Could not correctly update Burst status and meta-data!") burst_entity.status = burst_status burst_entity.error_message = "Error when updating Burst Status" burst_entity.finish_time = datetime.now() dao.store_entity(burst_entity) def persist_operation_state(self, operation, operation_status, message=None): """ Update Operation instance state. Store it in DB and on HDD/ :param operation: Operation instance :param operation_status: new status :param message: message in case of error :return: operation instance changed """ operation.mark_complete(operation_status, message) dao.store_entity(operation) operation = dao.get_operation_by_id(operation.id) self.file_helper.write_operation_metadata(operation) return operation def get_burst_for_operation_id(self, operation_id): return dao.get_burst_for_operation_id(operation_id) @staticmethod def rename_burst(burst_id, new_name): """ Rename the burst given by burst_id, setting it's new name to burst_name. """ burst = dao.get_burst_by_id(burst_id) burst.name = new_name dao.store_entity(burst) @staticmethod def get_available_bursts(project_id): """ Return all the burst for the current project. """ bursts = dao.get_bursts_for_project( project_id, page_size=MAX_BURSTS_DISPLAYED) or [] # for burst in bursts: # burst.prepare_after_load() return bursts @staticmethod def populate_burst_disk_usage(bursts): """ Adds a disk_usage field to each burst object. The disk usage is computed as the sum of the datatypes generated by a burst """ sizes = dao.compute_bursts_disk_size([b.id for b in bursts]) for b in bursts: b.disk_size = format_bytes_human(sizes[b.id]) def update_history_status(self, id_list): """ For each burst_id received in the id_list read new status from DB and return a list [id, new_status] pair. """ result = [] for b_id in id_list: burst = dao.get_burst_by_id(b_id) # burst.prepare_after_load() if burst is not None: if burst.status == burst.BURST_RUNNING: running_time = datetime.now() - burst.start_time else: running_time = burst.finish_time - burst.start_time running_time = format_timedelta(running_time, most_significant2=False) if burst.status == burst.BURST_ERROR: msg = 'Check Operations page for error Message' else: msg = '' result.append([ burst.id, burst.status, burst.is_group, msg, running_time ]) else: self.logger.debug("Could not find burst with id=" + str(b_id) + ". Might have been deleted by user!!") return result def stop_burst(self, burst): raise NotImplementedError def cancel_or_remove_burst(self, burst_id): raise NotImplementedError @staticmethod def update_simulation_fields(burst_id, op_simulation_id, simulation_gid): burst = dao.get_burst_by_id(burst_id) burst.fk_simulation_id = op_simulation_id burst.simulator_gid = simulation_gid.hex dao.store_entity(burst)
class OperationService: """ Class responsible for preparing an operation launch. It will prepare parameters, and decide if the operation is to be executed immediately, or to be sent on the cluster. """ ATT_UID = "uid" def __init__(self): self.logger = get_logger(self.__class__.__module__) self.workflow_service = WorkflowService() self.file_helper = FilesHelper() ########################################################################################## ######## Methods related to launching operations start here ############################## ########################################################################################## def initiate_operation(self, current_user, project_id, adapter_instance, temporary_storage, visible=True, **kwargs): """ Gets the parameters of the computation from the previous inputs form, and launches a computation (on the cluster or locally). Invoke custom method on an Adapter Instance. Make sure when the operation has finished that the correct results are stored into DB. """ if not isinstance(adapter_instance, ABCAdapter): self.logger.warning("Inconsistent Adapter Class:" + str(adapter_instance.__class__)) raise LaunchException("Developer Exception!!") # Prepare Files parameters files = {} kw2 = copy(kwargs) for i, j in six.iteritems(kwargs): if isinstance(j, FieldStorage) or isinstance(j, Part): files[i] = j del kw2[i] temp_files = {} try: for i, j in six.iteritems(files): if j.file is None: kw2[i] = None continue uq_name = utils.date2string(datetime.now(), True) + '_' + str(i) # We have to add original file name to end, in case file processing # involves file extension reading file_name = TEMPORARY_PREFIX + uq_name + '_' + j.filename file_name = os.path.join(temporary_storage, file_name) kw2[i] = file_name temp_files[i] = file_name with open(file_name, 'wb') as file_obj: file_obj.write(j.file.read()) self.logger.debug("Will store file:" + file_name) kwargs = kw2 except Exception as excep: self._handle_exception(excep, temp_files, "Could not launch operation: invalid input files!") ### Store Operation entity. algo = adapter_instance.stored_adapter algo_category = dao.get_category_by_id(algo.fk_category) operations = self.prepare_operations(current_user.id, project_id, algo, algo_category, {}, visible, **kwargs)[0] if isinstance(adapter_instance, ABCSynchronous): if len(operations) > 1: raise LaunchException("Synchronous operations are not supporting ranges!") if len(operations) < 1: self.logger.warning("No operation was defined") raise LaunchException("Invalid empty Operation!!!") return self.initiate_prelaunch(operations[0], adapter_instance, temp_files, **kwargs) else: return self._send_to_cluster(operations, adapter_instance, current_user.username) @staticmethod def _prepare_metadata(initial_metadata, algo_category, operation_group, submit_data): """ Gather metadata from submitted fields and current to be execute algorithm. Will populate STATE, GROUP in metadata """ metadata = copy(initial_metadata) user_group = None if DataTypeMetaData.KEY_OPERATION_TAG in submit_data: user_group = submit_data[DataTypeMetaData.KEY_OPERATION_TAG] if operation_group is not None: metadata[DataTypeMetaData.KEY_OPERATION_TAG] = operation_group.name if DataTypeMetaData.KEY_TAG_1 in submit_data: metadata[DataTypeMetaData.KEY_TAG_1] = submit_data[DataTypeMetaData.KEY_TAG_1] metadata[DataTypeMetaData.KEY_STATE] = algo_category.defaultdatastate return metadata, user_group @staticmethod def _read_set(values): """ Parse a committed UI possible list of values, into a set converted into string.""" if isinstance(values, list): set_values = [] values_str = "" for val in values: if val not in set_values: set_values.append(val) values_str = values_str + " " + str(val) values = values_str return str(values).strip() def group_operation_launch(self, user_id, project_id, algorithm_id, category_id, existing_dt_group=None, **kwargs): """ Create and prepare the launch of a group of operations. """ category = dao.get_category_by_id(category_id) algorithm = dao.get_algorithm_by_id(algorithm_id) ops, _ = self.prepare_operations(user_id, project_id, algorithm, category, {}, existing_dt_group=existing_dt_group, **kwargs) for operation in ops: self.launch_operation(operation.id, True) def prepare_operations(self, user_id, project_id, algorithm, category, metadata, visible=True, existing_dt_group=None, **kwargs): """ Do all the necessary preparations for storing an operation. If it's the case of a range of values create an operation group and multiple operations for each possible instance from the range. :param metadata: Initial MetaData with potential Burst identification inside. """ operations = [] available_args, group = self._prepare_group(project_id, existing_dt_group, kwargs) if len(available_args) > TvbProfile.current.MAX_RANGE_NUMBER: raise LaunchException("Too big range specified. You should limit the" " resulting operations to %d" % TvbProfile.current.MAX_RANGE_NUMBER) else: self.logger.debug("Launching a range with %d operations..." % len(available_args)) group_id = None if group is not None: group_id = group.id metadata, user_group = self._prepare_metadata(metadata, category, group, kwargs) self.logger.debug("Saving Operation(userId=" + str(user_id) + ",projectId=" + str(project_id) + "," + str(metadata) + ",algorithmId=" + str(algorithm.id) + ", ops_group= " + str(group_id) + ")") visible_operation = visible and category.display is False meta_str = json.dumps(metadata) for (one_set_of_args, range_vals) in available_args: range_values = json.dumps(range_vals) if range_vals else None operation = model.Operation(user_id, project_id, algorithm.id, json.dumps(one_set_of_args, cls=MapAsJson.MapAsJsonEncoder), meta_str, op_group_id=group_id, user_group=user_group, range_values=range_values) operation.visible = visible_operation operations.append(operation) operations = dao.store_entities(operations) if group is not None: burst_id = None if DataTypeMetaData.KEY_BURST in metadata: burst_id = metadata[DataTypeMetaData.KEY_BURST] if existing_dt_group is None: datatype_group = model.DataTypeGroup(group, operation_id=operations[0].id, fk_parent_burst=burst_id, state=metadata[DataTypeMetaData.KEY_STATE]) dao.store_entity(datatype_group) else: # Reset count existing_dt_group.count_results = None dao.store_entity(existing_dt_group) return operations, group def prepare_operations_for_workflowsteps(self, workflow_step_list, workflows, user_id, burst_id, project_id, group, sim_operations): """ Create and store Operation entities from a list of Workflow Steps. Will be generated workflows x workflow_step_list Operations. For every step in workflow_step_list one OperationGroup and one DataTypeGroup will be created (in case of PSE). """ for step in workflow_step_list: operation_group = None if (group is not None) and not isinstance(step, model.WorkflowStepView): operation_group = model.OperationGroup(project_id=project_id, ranges=group.range_references) operation_group = dao.store_entity(operation_group) operation = None metadata = {DataTypeMetaData.KEY_BURST: burst_id} algo_category = dao.get_algorithm_by_id(step.fk_algorithm) if algo_category is not None: algo_category = algo_category.algorithm_category for wf_idx, workflow in enumerate(workflows): cloned_w_step = step.clone() cloned_w_step.fk_workflow = workflow.id dynamic_params = cloned_w_step.dynamic_param op_params = cloned_w_step.static_param op_params.update(dynamic_params) range_values = None group_id = None if operation_group is not None: group_id = operation_group.id range_values = sim_operations[wf_idx].range_values if not isinstance(step, model.WorkflowStepView): ## For visualization steps, do not create operations, as those are not really needed. metadata, user_group = self._prepare_metadata(metadata, algo_category, operation_group, op_params) operation = model.Operation(user_id, project_id, step.fk_algorithm, json.dumps(op_params, cls=MapAsJson.MapAsJsonEncoder), meta=json.dumps(metadata), op_group_id=group_id, range_values=range_values, user_group=user_group) operation.visible = step.step_visible operation = dao.store_entity(operation) cloned_w_step.fk_operation = operation.id dao.store_entity(cloned_w_step) if operation_group is not None and operation is not None: datatype_group = model.DataTypeGroup(operation_group, operation_id=operation.id, fk_parent_burst=burst_id, state=metadata[DataTypeMetaData.KEY_STATE]) dao.store_entity(datatype_group) def initiate_prelaunch(self, operation, adapter_instance, temp_files, **kwargs): """ Public method. This should be the common point in calling an adapter- method. """ result_msg = "" try: unique_id = None if self.ATT_UID in kwargs: unique_id = kwargs[self.ATT_UID] filtered_kwargs = adapter_instance.prepare_ui_inputs(kwargs) self.logger.debug("Launching operation " + str(operation.id) + " with " + str(filtered_kwargs)) operation = dao.get_operation_by_id(operation.id) # Load Lazy fields params = dict() for k, value_ in filtered_kwargs.items(): params[str(k)] = value_ disk_space_per_user = TvbProfile.current.MAX_DISK_SPACE pending_op_disk_space = dao.compute_disk_size_for_started_ops(operation.fk_launched_by) user_disk_space = dao.compute_user_generated_disk_size(operation.fk_launched_by) # From kB to Bytes available_space = disk_space_per_user - pending_op_disk_space - user_disk_space result_msg, nr_datatypes = adapter_instance._prelaunch(operation, unique_id, available_space, **params) operation = dao.get_operation_by_id(operation.id) ## Update DB stored kwargs for search purposes, to contain only valuable params (no unselected options) operation.parameters = json.dumps(kwargs) operation.mark_complete(model.STATUS_FINISHED) if nr_datatypes > 0: #### Write operation meta-XML only if some result are returned self.file_helper.write_operation_metadata(operation) dao.store_entity(operation) self._remove_files(temp_files) except zipfile.BadZipfile as excep: msg = "The uploaded file is not a valid ZIP!" self._handle_exception(excep, temp_files, msg, operation) except TVBException as excep: self._handle_exception(excep, temp_files, excep.message, operation) except MemoryError: msg = ("Could not execute operation because there is not enough free memory." + " Please adjust operation parameters and re-launch it.") self._handle_exception(Exception(msg), temp_files, msg, operation) except Exception as excep1: msg = "Could not launch Operation with the given input data!" self._handle_exception(excep1, temp_files, msg, operation) ### Try to find next workflow Step. It might throw WorkflowException next_op_id = self.workflow_service.prepare_next_step(operation.id) self.launch_operation(next_op_id) return result_msg def _send_to_cluster(self, operations, adapter_instance, current_username="******"): """ Initiate operation on cluster""" for operation in operations: try: BACKEND_CLIENT.execute(str(operation.id), current_username, adapter_instance) except Exception as excep: self._handle_exception(excep, {}, "Could not start operation!", operation) return operations def launch_operation(self, operation_id, send_to_cluster=False, adapter_instance=None): """ Method exposed for Burst-Workflow related calls. It is used for cascading operation in the same workflow. """ if operation_id is not None: operation = dao.get_operation_by_id(operation_id) if adapter_instance is None: algorithm = operation.algorithm adapter_instance = ABCAdapter.build_adapter(algorithm) parsed_params = utils.parse_json_parameters(operation.parameters) if send_to_cluster: self._send_to_cluster([operation], adapter_instance, operation.user.username) else: self.initiate_prelaunch(operation, adapter_instance, {}, **parsed_params) def _handle_exception(self, exception, temp_files, message, operation=None): """ Common way to treat exceptions: - remove temporary files, if any - set status ERROR on current operation (if any) - log exception """ self.logger.exception(message) if operation is not None: self.workflow_service.persist_operation_state(operation, model.STATUS_ERROR, unicode(exception)) self.workflow_service.update_executed_workflow_state(operation) self._remove_files(temp_files) exception.message = message raise exception, None, sys.exc_info()[2] # when rethrowing in python this is required to preserve the stack trace def _remove_files(self, file_dictionary): """ Remove any files that exist in the file_dictionary. Currently used to delete temporary files created during an operation. """ for pth in file_dictionary.itervalues(): pth = str(pth) try: if os.path.exists(pth) and os.path.isfile(pth): os.remove(pth) self.logger.debug("We no longer need file:" + pth + " => deleted") else: self.logger.warning("Trying to remove not existent file:" + pth) except OSError: self.logger.exception("Could not cleanup file!") @staticmethod def _range_name(range_no): return model.PARAM_RANGE_PREFIX + str(range_no) def _prepare_group(self, project_id, existing_dt_group, kwargs): """ Create and store OperationGroup entity, or return None """ # Standard ranges as accepted from UI range1_values = self.get_range_values(kwargs, self._range_name(1)) range2_values = self.get_range_values(kwargs, self._range_name(2)) available_args = self.__expand_arguments([(kwargs, None)], range1_values, self._range_name(1)) available_args = self.__expand_arguments(available_args, range2_values, self._range_name(2)) is_group = False ranges = [] if self._range_name(1) in kwargs and range1_values is not None: is_group = True ranges.append(json.dumps((kwargs[self._range_name(1)], range1_values))) if self._range_name(2) in kwargs and range2_values is not None: is_group = True ranges.append(json.dumps((kwargs[self._range_name(2)], range2_values))) # Now for additional ranges which might be the case for the 'model exploration' last_range_idx = 3 ranger_name = self._range_name(last_range_idx) while ranger_name in kwargs: values_for_range = self.get_range_values(kwargs, ranger_name) available_args = self.__expand_arguments(available_args, values_for_range, ranger_name) last_range_idx += 1 ranger_name = self._range_name(last_range_idx) if last_range_idx > 3: ranges = [] # Since we only have 3 fields in db for this just hide it if not is_group: group = None elif existing_dt_group is None: group = model.OperationGroup(project_id=project_id, ranges=ranges) group = dao.store_entity(group) else: group = existing_dt_group.parent_operation_group return available_args, group def get_range_values(self, kwargs, ranger_name): """ For the ranger given by ranger_name look in kwargs and return the array with all the possible values. """ if ranger_name not in kwargs: return None if str(kwargs[ranger_name]) not in kwargs: return None range_values = [] try: range_data = json.loads(str(kwargs[str(kwargs[ranger_name])])) except Exception: try: range_data = [x.strip() for x in str(kwargs[str(kwargs[ranger_name])]).split(',') if len(x.strip()) > 0] return range_data except Exception: self.logger.exception("Could not launch operation !") raise LaunchException("Could not launch with no data from:" + str(ranger_name)) if type(range_data) in (list, tuple): return range_data if (constants.ATT_MINVALUE in range_data) and (constants.ATT_MAXVALUE in range_data): lo_val = float(range_data[constants.ATT_MINVALUE]) hi_val = float(range_data[constants.ATT_MAXVALUE]) step = float(range_data[constants.ATT_STEP]) range_values = list(Range(lo=lo_val, hi=hi_val, step=step, mode=Range.MODE_INCLUDE_BOTH)) else: for possible_value in range_data: if range_data[possible_value]: range_values.append(possible_value) return range_values @staticmethod def __expand_arguments(arguments_list, range_values, range_title): """ Parse the arguments submitted from UI (flatten form) If any ranger is found, return a list of arguments for all possible operations. """ if range_values is None: return arguments_list result = [] for value in range_values: for args, range_ in arguments_list: kw_new = copy(args) range_new = copy(range_) kw_new[kw_new[range_title]] = value if range_new is None: range_new = {} range_new[kw_new[range_title]] = value del kw_new[range_title] result.append((kw_new, range_new)) return result ########################################################################################## ######## Methods related to stopping and restarting operations start here ################ ########################################################################################## def stop_operation(self, operation_id): """ Stop the operation given by the operation id. """ return BACKEND_CLIENT.stop_operation(int(operation_id))
class WorkflowService: """ service layer for work-flow entity. """ def __init__(self): self.logger = get_logger(self.__class__.__module__) self.file_helper = FilesHelper() def persist_operation_state(self, operation, operation_status, message=None): """ Update Operation instance state. Store it in DB and on HDD/ :param operation: Operation instance :param operation_status: new status :param message: message in case of error :return: operation instance changed """ operation.mark_complete(operation_status, unicode(message)) dao.store_entity(operation) operation = dao.get_operation_by_id(operation.id) self.file_helper.write_operation_metadata(operation) return operation @staticmethod def store_workflow_step(workflow_step): """ Store a workflow step entity. """ dao.store_entity(workflow_step) @staticmethod def create_and_store_workflow(project_id, burst_id, simulator_index, simulator_id, operations): """ Create and store the workflow given the project, user and burst in which the workflow is created. :param simulator_index: the index of the simulator in the workflow :param simulator_id: the id of the simulator adapter :param operations: a list with the operations created for the simulator steps """ workflows = [] for operation in operations: new_workflow = model.Workflow(project_id, burst_id) new_workflow = dao.store_entity(new_workflow) workflows.append(new_workflow) simulation_step = model.WorkflowStep(algorithm_id=simulator_id, workflow_id=new_workflow.id, step_index=simulator_index, static_param=operation.parameters) simulation_step.fk_operation = operation.id dao.store_entity(simulation_step) return workflows @staticmethod def set_dynamic_step_references(workflow_step, step_reference): """ :param workflow_step: a valid instance of a workflow_step :param step_reference: the step to which every dataType reference index should be set For each dynamic parameter of the given workflow_step set the 'step_index' at step_reference. """ dynamic_params = workflow_step.dynamic_param for entry in dynamic_params: dynamic_params[entry][WorkflowStepConfiguration.STEP_INDEX_KEY] = step_reference workflow_step.dynamic_param = dynamic_params def prepare_next_step(self, last_executed_op_id): """ If the operation with id 'last_executed_op_id' resulted after the execution of a workflow step then this method will launch the operation corresponding to the next step from the workflow. """ try: current_step, next_workflow_step = self._get_data(last_executed_op_id) if next_workflow_step is not None: operation = dao.get_operation_by_id(next_workflow_step.fk_operation) dynamic_param_names = next_workflow_step.dynamic_workflow_param_names if len(dynamic_param_names) > 0: op_params = json.loads(operation.parameters) for param_name in dynamic_param_names: dynamic_param = op_params[param_name] former_step = dao.get_workflow_step_by_step_index(next_workflow_step.fk_workflow, dynamic_param[ WorkflowStepConfiguration.STEP_INDEX_KEY]) if type(dynamic_param[WorkflowStepConfiguration.DATATYPE_INDEX_KEY]) is IntType: datatypes = dao.get_results_for_operation(former_step.fk_operation) op_params[param_name] = datatypes[ dynamic_param[WorkflowStepConfiguration.DATATYPE_INDEX_KEY]].gid else: previous_operation = dao.get_operation_by_id(former_step.fk_operation) op_params[param_name] = json.loads(previous_operation.parameters)[ dynamic_param[WorkflowStepConfiguration.DATATYPE_INDEX_KEY]] operation.parameters = json.dumps(op_params) operation = dao.store_entity(operation) return operation.id elif current_step is not None: current_workflow = dao.get_workflow_by_id(current_step.fk_workflow) current_workflow.status = current_workflow.STATUS_FINISHED dao.store_entity(current_workflow) burst_entity = dao.get_burst_by_id(current_workflow.fk_burst) parallel_workflows = dao.get_workflows_for_burst(burst_entity.id) all_finished = True for workflow in parallel_workflows: if workflow.status == workflow.STATUS_STARTED: all_finished = False if all_finished: self.mark_burst_finished(burst_entity) return None except Exception, excep: self.logger.error(excep) self.logger.exception(excep) raise WorkflowInterStepsException(excep)
class FilesHelperTest(TransactionalTestCase): """ This class contains tests for the tvb.core.entities.file.files_helper module. """ PROJECT_NAME = "test_proj" def setUp(self): """ Set up the context needed by the tests. """ self.files_helper = FilesHelper() self.test_user = TestFactory.create_user() self.test_project = TestFactory.create_project(self.test_user, self.PROJECT_NAME) def tearDown(self): """ Remove generated project during tests. """ self.delete_project_folders() def test_check_created(self): """ Test standard flows for check created. """ self.files_helper.check_created() self.assertTrue(os.path.exists(root_storage), "Storage not created!") self.files_helper.check_created(os.path.join(root_storage, "test")) self.assertTrue(os.path.exists(root_storage), "Storage not created!") self.assertTrue(os.path.exists(os.path.join(root_storage, "test")), "Test directory not created!") def test_get_project_folder(self): """ Test the get_project_folder method which should create a folder in case it doesn't already exist. """ project_path = self.files_helper.get_project_folder(self.test_project) self.assertTrue(os.path.exists(project_path), "Folder doesn't exist") folder_path = self.files_helper.get_project_folder( self.test_project, "43") self.assertTrue(os.path.exists(project_path), "Folder doesn't exist") self.assertTrue(os.path.exists(folder_path), "Folder doesn't exist") def test_rename_project_structure(self): """ Try to rename the folder structure of a project. Standard flow. """ self.files_helper.get_project_folder(self.test_project) path, name = self.files_helper.rename_project_structure( self.test_project.name, "new_name") self.assertNotEqual(path, name, "Rename didn't take effect.") def test_rename_structure_same_name(self): """ Try to rename the folder structure of a project. Same name. """ self.files_helper.get_project_folder(self.test_project) self.assertRaises(FileStructureException, self.files_helper.rename_project_structure, self.test_project.name, self.PROJECT_NAME) def test_remove_project_structure(self): """ Check that remove project structure deletes the corresponding folder. Standard flow. """ full_path = self.files_helper.get_project_folder(self.test_project) self.assertTrue(os.path.exists(full_path), "Folder was not created.") self.files_helper.remove_project_structure(self.test_project.name) self.assertFalse(os.path.exists(full_path), "Project folder not deleted.") def test_write_project_metadata(self): """ Write XML for test-project. """ self.files_helper.write_project_metadata(self.test_project) expected_file = self.files_helper.get_project_meta_file_path( self.PROJECT_NAME) self.assertTrue(os.path.exists(expected_file)) project_meta = XMLReader(expected_file).read_metadata() loaded_project = model.Project(None, None) loaded_project.from_dict(project_meta, self.test_user.id) self.assertEqual(self.test_project.name, loaded_project.name) self.assertEqual(self.test_project.description, loaded_project.description) self.assertEqual(self.test_project.gid, loaded_project.gid) expected_dict = self.test_project.to_dict()[1] del expected_dict['last_updated'] found_dict = loaded_project.to_dict()[1] del found_dict['last_updated'] self.assertDictContainsSubset(expected_dict, found_dict) self.assertDictContainsSubset(found_dict, expected_dict) def test_write_operation_metadata(self): """ Test that a correct XML is created for an operation. """ operation = TestFactory.create_operation( test_user=self.test_user, test_project=self.test_project) expected_file = self.files_helper.get_operation_meta_file_path( self.PROJECT_NAME, operation.id) self.assertFalse(os.path.exists(expected_file)) self.files_helper.write_operation_metadata(operation) self.assertTrue(os.path.exists(expected_file)) operation_meta = XMLReader(expected_file).read_metadata() loaded_operation = model.Operation(None, None, None, None) loaded_operation.from_dict(operation_meta, dao) expected_dict = operation.to_dict()[1] found_dict = loaded_operation.to_dict()[1] for key, value in expected_dict.iteritems(): self.assertEqual(str(value), str(found_dict[key])) # Now validate that operation metaData can be also updated self.assertNotEqual("new_group_name", found_dict['user_group']) self.files_helper.update_operation_metadata(self.PROJECT_NAME, "new_group_name", operation.id) found_dict = XMLReader(expected_file).read_metadata() self.assertEqual("new_group_name", found_dict['user_group']) def test_remove_dt_happy_flow(self): """ Happy flow for removing a file related to a DataType. """ folder_path = self.files_helper.get_project_folder( self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path open(datatype.get_storage_file_path(), 'w') self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") self.files_helper.remove_datatype(datatype) self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not deleted!") def test_remove_dt_non_existent(self): """ Try to call remove on a dataType with no H5 file. Should work. """ folder_path = self.files_helper.get_project_folder( self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path self.assertFalse(os.path.exists(datatype.get_storage_file_path())) self.files_helper.remove_datatype(datatype) def test_move_datatype(self): """ Make sure associated H5 file is moved to a correct new location. """ folder_path = self.files_helper.get_project_folder( self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path open(datatype.get_storage_file_path(), 'w') self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") self.files_helper.move_datatype(datatype, self.PROJECT_NAME + '11', "43") self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not moved!") datatype.storage_path = self.files_helper.get_project_folder( self.PROJECT_NAME + '11', "43") self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") def test_find_relative_path(self): """ Tests that relative path is computed properly. """ rel_path = self.files_helper.find_relative_path( "/root/up/to/here/test/it/now", "/root/up/to/here") self.assertEqual(rel_path, os.sep.join(["test", "it", "now"]), "Did not extract relative path as expected.") def test_remove_files_valid(self): """ Pass a valid list of files and check they are all removed. """ file_list = ["test1", "test2", "test3"] for file_n in file_list: fp = open(file_n, 'w') fp.write('test') fp.close() for file_n in file_list: self.assertTrue(os.path.isfile(file_n)) self.files_helper.remove_files(file_list) for file_n in file_list: self.assertFalse(os.path.isfile(file_n)) def test_remove_folder(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" os.mkdir(folder_name) self.assertTrue(os.path.isdir(folder_name), "Folder should be created.") self.files_helper.remove_folder(folder_name) self.assertFalse(os.path.isdir(folder_name), "Folder should be deleted.") def test_remove_folder_non_existing_ignore_exc(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.") self.files_helper.remove_folder(folder_name, ignore_errors=True) def test_remove_folder_non_existing(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.") self.assertRaises(FileStructureException, self.files_helper.remove_folder, folder_name, False)
class WorkflowService: """ service layer for work-flow entity. """ def __init__(self): self.logger = get_logger(self.__class__.__module__) self.file_helper = FilesHelper() def persist_operation_state(self, operation, operation_status, message=None): """ Update Operation instance state. Store it in DB and on HDD/ :param operation: Operation instance :param operation_status: new status :param message: message in case of error :return: operation instance changed """ operation.mark_complete(operation_status, unicode(message)) dao.store_entity(operation) operation = dao.get_operation_by_id(operation.id) self.file_helper.write_operation_metadata(operation) return operation @staticmethod def store_workflow_step(workflow_step): """ Store a workflow step entity. """ dao.store_entity(workflow_step) @staticmethod def create_and_store_workflow(project_id, burst_id, simulator_index, simulator_id, operations): """ Create and store the workflow given the project, user and burst in which the workflow is created. :param simulator_index: the index of the simulator in the workflow :param simulator_id: the id of the simulator adapter :param operations: a list with the operations created for the simulator steps """ workflows = [] for operation in operations: new_workflow = model.Workflow(project_id, burst_id) new_workflow = dao.store_entity(new_workflow) workflows.append(new_workflow) simulation_step = model.WorkflowStep(algorithm_id=simulator_id, workflow_id=new_workflow.id, step_index=simulator_index, static_param=operation.parameters) simulation_step.fk_operation = operation.id dao.store_entity(simulation_step) return workflows @staticmethod def set_dynamic_step_references(workflow_step, step_reference): """ :param workflow_step: a valid instance of a workflow_step :param step_reference: the step to which every dataType reference index should be set For each dynamic parameter of the given workflow_step set the 'step_index' at step_reference. """ dynamic_params = workflow_step.dynamic_param for entry in dynamic_params: dynamic_params[entry][WorkflowStepConfiguration.STEP_INDEX_KEY] = step_reference workflow_step.dynamic_param = dynamic_params def prepare_next_step(self, last_executed_op_id): """ If the operation with id 'last_executed_op_id' resulted after the execution of a workflow step then this method will launch the operation corresponding to the next step from the workflow. """ try: current_step, next_workflow_step = self._get_data(last_executed_op_id) if next_workflow_step is not None: operation = dao.get_operation_by_id(next_workflow_step.fk_operation) dynamic_param_names = next_workflow_step.dynamic_workflow_param_names if len(dynamic_param_names) > 0: op_params = json.loads(operation.parameters) for param_name in dynamic_param_names: dynamic_param = op_params[param_name] former_step = dao.get_workflow_step_by_step_index(next_workflow_step.fk_workflow, dynamic_param[ WorkflowStepConfiguration.STEP_INDEX_KEY]) if type(dynamic_param[WorkflowStepConfiguration.DATATYPE_INDEX_KEY]) is IntType: datatypes = dao.get_results_for_operation(former_step.fk_operation) op_params[param_name] = datatypes[ dynamic_param[WorkflowStepConfiguration.DATATYPE_INDEX_KEY]].gid else: previous_operation = dao.get_operation_by_id(former_step.fk_operation) op_params[param_name] = json.loads(previous_operation.parameters)[ dynamic_param[WorkflowStepConfiguration.DATATYPE_INDEX_KEY]] operation.parameters = json.dumps(op_params) operation = dao.store_entity(operation) return operation.id elif current_step is not None: current_workflow = dao.get_workflow_by_id(current_step.fk_workflow) current_workflow.status = current_workflow.STATUS_FINISHED dao.store_entity(current_workflow) burst_entity = dao.get_burst_by_id(current_workflow.fk_burst) parallel_workflows = dao.get_workflows_for_burst(burst_entity.id) all_finished = True for workflow in parallel_workflows: if workflow.status == workflow.STATUS_STARTED: all_finished = False if all_finished: self.mark_burst_finished(burst_entity) return None except Exception as excep: self.logger.error(excep) self.logger.exception(excep) raise WorkflowInterStepsException(excep) def update_executed_workflow_state(self, operation): """ Used for updating the state of an executed workflow. Only if the operation with the specified id has resulted after the execution of an ExecutedWorkflowStep than the state of the ExecutedWorkflow to which belongs the step will be updated. """ executed_step, _ = self._get_data(operation.id) if executed_step is not None: if operation.status == model.STATUS_ERROR: all_executed_steps = dao.get_workflow_steps(executed_step.fk_workflow) for step in all_executed_steps: if step.step_index > executed_step.step_index: self.logger.debug("Marking unreached operation %s with error." % step.fk_operation) unreached_operation = dao.get_operation_by_id(step.fk_operation) self.persist_operation_state(unreached_operation, model.STATUS_ERROR, "Blocked by failure in step %s with message: \n\n%s." % ( executed_step.step_index, operation.additional_info)) workflow = dao.get_workflow_by_id(executed_step.fk_workflow) burst = dao.get_burst_by_id(workflow.fk_burst) self.mark_burst_finished(burst, error_message=operation.additional_info) dao.store_entity(burst) @staticmethod def _get_data(operation_id): """ For a given operation id, return the corresponding WorkflowStep and the NextWorkflowStep to be executed. """ executed_step = dao.get_workflow_step_for_operation(operation_id) if executed_step is not None: next_workflow_step = dao.get_workflow_step_by_step_index(executed_step.fk_workflow, executed_step.step_index + 1) return executed_step, next_workflow_step else: return None, None def mark_burst_finished(self, burst_entity, burst_status=None, error_message=None): """ Mark Burst status field. Also compute 'weight' for current burst: no of operations inside, estimate time on disk... :param burst_entity: BurstConfiguration to be updated, at finish time. :param burst_status: BurstConfiguration status. By default BURST_FINISHED :param error_message: If given, set the status to error and perpetuate the message. """ if burst_status is None: burst_status = model.BurstConfiguration.BURST_FINISHED if error_message is not None: burst_status = model.BurstConfiguration.BURST_ERROR try: ### If there are any DataType Groups in current Burst, update their counter. burst_dt_groups = dao.get_generic_entity(model.DataTypeGroup, burst_entity.id, "fk_parent_burst") for dt_group in burst_dt_groups: dt_group.count_results = dao.count_datatypes_in_group(dt_group.id) dt_group.disk_size, dt_group.subject = dao.get_summary_for_group(dt_group.id) dao.store_entity(dt_group) ### Update actual Burst entity fields burst_entity.datatypes_number = dao.count_datatypes_in_burst(burst_entity.id) burst_entity.workflows_number = dao.get_workflows_for_burst(burst_entity.id, is_count=True) burst_entity.status = burst_status burst_entity.error_message = error_message burst_entity.finish_time = datetime.now() dao.store_entity(burst_entity) except Exception: self.logger.exception("Could not correctly update Burst status and meta-data!") burst_entity.status = burst_status burst_entity.error_message = "Error when updating Burst Status" burst_entity.finish_time = datetime.now() dao.store_entity(burst_entity)
def _adapt_simulation_monitor_params(): """ For previous simulation with EEG monitor, adjust the change of input parameters. """ session = SA_SESSIONMAKER() param_connectivity = "connectivity" param_eeg_proj_old = "monitors_parameters_option_EEG_projection_matrix_data" param_eeg_proj_new = "monitors_parameters_option_EEG_projection" param_eeg_sensors = "monitors_parameters_option_EEG_sensors" param_eeg_rm = "monitors_parameters_option_EEG_region_mapping" try: all_eeg_ops = session.query(model.Operation).filter( model.Operation.parameters.ilike('%"' + param_eeg_proj_old + '"%')).all() files_helper = FilesHelper() all_bursts = dict() for eeg_op in all_eeg_ops: try: op_params = parse_json_parameters(eeg_op.parameters) LOGGER.debug("Updating " + str(op_params)) old_projection_guid = op_params[param_eeg_proj_old] connectivity_guid = op_params[param_connectivity] rm = dao.get_generic_entity(RegionMapping, connectivity_guid, "_connectivity")[0] dt = dao.get_generic_entity(model.DataType, old_projection_guid, "gid")[0] if dt.type == 'ProjectionSurfaceEEG': LOGGER.debug("Previous Prj is surface: " + old_projection_guid) new_projection_guid = old_projection_guid else: new_projection_guid = session.execute(text("""SELECT DT.gid FROM "MAPPED_PROJECTION_MATRIX_DATA" PMO, "DATA_TYPES" DTO, "MAPPED_PROJECTION_MATRIX_DATA" PM, "DATA_TYPES" DT WHERE DTO.id=PMO.id and DT.id=PM.id and PM._sensors=PMO._sensors and PM._sources='""" + rm._surface + """' and DTO.gid='""" + old_projection_guid + """';""")).fetchall()[0][0] LOGGER.debug("New Prj is surface: " + str(new_projection_guid)) sensors_guid = session.execute(text("""SELECT _sensors FROM "MAPPED_PROJECTION_MATRIX_DATA" WHERE id = '""" + str(dt.id) + """';""")).fetchall()[0][0] del op_params[param_eeg_proj_old] op_params[param_eeg_proj_new] = str(new_projection_guid) op_params[param_eeg_sensors] = str(sensors_guid) op_params[param_eeg_rm] = str(rm.gid) eeg_op.parameters = json.dumps(op_params, cls=MapAsJson.MapAsJsonEncoder) LOGGER.debug("New params:" + eeg_op.parameters) files_helper.write_operation_metadata(eeg_op) burst = dao.get_burst_for_operation_id(eeg_op.id) if burst is not None: LOGGER.debug("Updating burst:" + str(burst)) burst.prepare_after_load() del burst.simulator_configuration[param_eeg_proj_old] burst.simulator_configuration[param_eeg_proj_new] = {'value': str(new_projection_guid)} burst.simulator_configuration[param_eeg_sensors] = {'value': str(sensors_guid)} burst.simulator_configuration[param_eeg_rm] = {'value': str(rm.gid)} burst._simulator_configuration = json.dumps(burst.simulator_configuration, cls=MapAsJson.MapAsJsonEncoder) if not all_bursts.has_key(burst.id): all_bursts[burst.id] = burst except Exception: LOGGER.exception("Could not process " + str(eeg_op)) session.add_all(all_eeg_ops) session.add_all(all_bursts.values()) session.commit() except Exception: LOGGER.exception("Could not update Simulation Monitor Params") finally: session.close()
class FilesHelperTest(TransactionalTestCase): """ This class contains tests for the tvb.core.entities.file.files_helper module. """ PROJECT_NAME = "test_proj" def setUp(self): """ Set up the context needed by the tests. """ self.files_helper = FilesHelper() self.test_user = TestFactory.create_user() self.test_project = TestFactory.create_project(self.test_user, self.PROJECT_NAME) def tearDown(self): """ Remove generated project during tests. """ self.delete_project_folders() def test_check_created(self): """ Test standard flows for check created. """ self.files_helper.check_created() self.assertTrue(os.path.exists(root_storage), "Storage not created!") self.files_helper.check_created(os.path.join(root_storage, "test")) self.assertTrue(os.path.exists(root_storage), "Storage not created!") self.assertTrue(os.path.exists(os.path.join(root_storage, "test")), "Test directory not created!") def test_get_project_folder(self): """ Test the get_project_folder method which should create a folder in case it doesn't already exist. """ project_path = self.files_helper.get_project_folder(self.test_project) self.assertTrue(os.path.exists(project_path), "Folder doesn't exist") folder_path = self.files_helper.get_project_folder(self.test_project, "43") self.assertTrue(os.path.exists(project_path), "Folder doesn't exist") self.assertTrue(os.path.exists(folder_path), "Folder doesn't exist") def test_rename_project_structure(self): """ Try to rename the folder structure of a project. Standard flow. """ self.files_helper.get_project_folder(self.test_project) path, name = self.files_helper.rename_project_structure(self.test_project.name, "new_name") self.assertNotEqual(path, name, "Rename didn't take effect.") def test_rename_structure_same_name(self): """ Try to rename the folder structure of a project. Same name. """ self.files_helper.get_project_folder(self.test_project) self.assertRaises(FileStructureException, self.files_helper.rename_project_structure, self.test_project.name, self.PROJECT_NAME) def test_remove_project_structure(self): """ Check that remove project structure deletes the corresponding folder. Standard flow. """ full_path = self.files_helper.get_project_folder(self.test_project) self.assertTrue(os.path.exists(full_path), "Folder was not created.") self.files_helper.remove_project_structure(self.test_project.name) self.assertFalse(os.path.exists(full_path), "Project folder not deleted.") def test_write_project_metadata(self): """ Write XML for test-project. """ self.files_helper.write_project_metadata(self.test_project) expected_file = self.files_helper.get_project_meta_file_path(self.PROJECT_NAME) self.assertTrue(os.path.exists(expected_file)) project_meta = XMLReader(expected_file).read_metadata() loaded_project = model.Project(None, None) loaded_project.from_dict(project_meta, self.test_user.id) self.assertEqual(self.test_project.name, loaded_project.name) self.assertEqual(self.test_project.description, loaded_project.description) self.assertEqual(self.test_project.gid, loaded_project.gid) expected_dict = self.test_project.to_dict()[1] del expected_dict['last_updated'] found_dict = loaded_project.to_dict()[1] del found_dict['last_updated'] self.assertDictContainsSubset(expected_dict, found_dict) self.assertDictContainsSubset(found_dict, expected_dict) def test_write_operation_metadata(self): """ Test that a correct XML is created for an operation. """ operation = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project) expected_file = self.files_helper.get_operation_meta_file_path(self.PROJECT_NAME, operation.id) self.assertFalse(os.path.exists(expected_file)) self.files_helper.write_operation_metadata(operation) self.assertTrue(os.path.exists(expected_file)) operation_meta = XMLReader(expected_file).read_metadata() loaded_operation = model.Operation(None, None, None, None) loaded_operation.from_dict(operation_meta, dao) expected_dict = operation.to_dict()[1] found_dict = loaded_operation.to_dict()[1] for key, value in expected_dict.iteritems(): self.assertEqual(str(value), str(found_dict[key])) # Now validate that operation metaData can be also updated self.assertNotEqual("new_group_name", found_dict['user_group']) self.files_helper.update_operation_metadata(self.PROJECT_NAME, "new_group_name", operation.id) found_dict = XMLReader(expected_file).read_metadata() self.assertEqual("new_group_name", found_dict['user_group']) def test_remove_dt_happy_flow(self): """ Happy flow for removing a file related to a DataType. """ folder_path = self.files_helper.get_project_folder(self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path open(datatype.get_storage_file_path(), 'w') self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") self.files_helper.remove_datatype(datatype) self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not deleted!") def test_remove_dt_non_existent(self): """ Try to call remove on a dataType with no H5 file. Should work. """ folder_path = self.files_helper.get_project_folder(self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path self.assertFalse(os.path.exists(datatype.get_storage_file_path())) self.files_helper.remove_datatype(datatype) def test_move_datatype(self): """ Make sure associated H5 file is moved to a correct new location. """ folder_path = self.files_helper.get_project_folder(self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path open(datatype.get_storage_file_path(), 'w') self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") self.files_helper.move_datatype(datatype, self.PROJECT_NAME + '11', "43") self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not moved!") datatype.storage_path = self.files_helper.get_project_folder(self.PROJECT_NAME + '11', "43") self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") def test_find_relative_path(self): """ Tests that relative path is computed properly. """ rel_path = self.files_helper.find_relative_path("/root/up/to/here/test/it/now", "/root/up/to/here") self.assertEqual(rel_path, os.sep.join(["test", "it", "now"]), "Did not extract relative path as expected.") def test_remove_files_valid(self): """ Pass a valid list of files and check they are all removed. """ file_list = ["test1", "test2", "test3"] for file_n in file_list: fp = open(file_n, 'w') fp.write('test') fp.close() for file_n in file_list: self.assertTrue(os.path.isfile(file_n)) self.files_helper.remove_files(file_list) for file_n in file_list: self.assertFalse(os.path.isfile(file_n)) def test_remove_folder(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" os.mkdir(folder_name) self.assertTrue(os.path.isdir(folder_name), "Folder should be created.") self.files_helper.remove_folder(folder_name) self.assertFalse(os.path.isdir(folder_name), "Folder should be deleted.") def test_remove_folder_non_existing_ignore_exc(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.") self.files_helper.remove_folder(folder_name, ignore_errors=True) def test_remove_folder_non_existing(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.") self.assertRaises(FileStructureException, self.files_helper.remove_folder, folder_name, False)
class WorkflowService: """ service layer for work-flow entity. """ def __init__(self): self.logger = get_logger(self.__class__.__module__) self.file_helper = FilesHelper() def persist_operation_state(self, operation, operation_status, message=None): """ Update Operation instance state. Store it in DB and on HDD/ :param operation: Operation instance :param operation_status: new status :param message: message in case of error :return: operation instance changed """ operation.mark_complete(operation_status, message) dao.store_entity(operation) operation = dao.get_operation_by_id(operation.id) self.file_helper.write_operation_metadata(operation) return operation @staticmethod def store_workflow_step(workflow_step): """ Store a workflow step entity. """ dao.store_entity(workflow_step) @staticmethod def create_and_store_workflow(project_id, burst_id, simulator_index, simulator_id, operations): """ Create and store the workflow given the project, user and burst in which the workflow is created. :param simulator_index: the index of the simulator in the workflow :param simulator_id: the id of the simulator adapter :param operations: a list with the operations created for the simulator steps """ workflows = [] for operation in operations: new_workflow = model.Workflow(project_id, burst_id) new_workflow = dao.store_entity(new_workflow) workflows.append(new_workflow) simulation_step = model.WorkflowStep( algorithm_id=simulator_id, workflow_id=new_workflow.id, step_index=simulator_index, static_param=operation.parameters) simulation_step.fk_operation = operation.id dao.store_entity(simulation_step) return workflows @staticmethod def set_dynamic_step_references(workflow_step, step_reference): """ :param workflow_step: a valid instance of a workflow_step :param step_reference: the step to which every dataType reference index should be set For each dynamic parameter of the given workflow_step set the 'step_index' at step_reference. """ dynamic_params = workflow_step.dynamic_param for entry in dynamic_params: dynamic_params[entry][ WorkflowStepConfiguration.STEP_INDEX_KEY] = step_reference workflow_step.dynamic_param = dynamic_params def prepare_next_step(self, last_executed_op_id): """ If the operation with id 'last_executed_op_id' resulted after the execution of a workflow step then this method will launch the operation corresponding to the next step from the workflow. """ try: current_step, next_workflow_step = self._get_data( last_executed_op_id) if next_workflow_step is not None: operation = dao.get_operation_by_id( next_workflow_step.fk_operation) dynamic_param_names = next_workflow_step.dynamic_workflow_param_names if len(dynamic_param_names) > 0: op_params = json.loads(operation.parameters) for param_name in dynamic_param_names: dynamic_param = op_params[param_name] former_step = dao.get_workflow_step_by_step_index( next_workflow_step.fk_workflow, dynamic_param[ WorkflowStepConfiguration.STEP_INDEX_KEY]) if type(dynamic_param[WorkflowStepConfiguration. DATATYPE_INDEX_KEY]) is int: datatypes = dao.get_results_for_operation( former_step.fk_operation) op_params[param_name] = datatypes[ dynamic_param[WorkflowStepConfiguration. DATATYPE_INDEX_KEY]].gid else: previous_operation = dao.get_operation_by_id( former_step.fk_operation) op_params[param_name] = json.loads( previous_operation.parameters)[ dynamic_param[WorkflowStepConfiguration. DATATYPE_INDEX_KEY]] operation.parameters = json.dumps(op_params) operation = dao.store_entity(operation) return operation.id elif current_step is not None: current_workflow = dao.get_workflow_by_id( current_step.fk_workflow) current_workflow.status = current_workflow.STATUS_FINISHED dao.store_entity(current_workflow) burst_entity = dao.get_burst_by_id(current_workflow.fk_burst) parallel_workflows = dao.get_workflows_for_burst( burst_entity.id) all_finished = True for workflow in parallel_workflows: if workflow.status == workflow.STATUS_STARTED: all_finished = False if all_finished: self.mark_burst_finished(burst_entity) return None except Exception as excep: self.logger.error(excep) self.logger.exception(excep) raise WorkflowInterStepsException(excep) def update_executed_workflow_state(self, operation): """ Used for updating the state of an executed workflow. Only if the operation with the specified id has resulted after the execution of an ExecutedWorkflowStep than the state of the ExecutedWorkflow to which belongs the step will be updated. """ executed_step, _ = self._get_data(operation.id) if executed_step is not None: if operation.status == model.STATUS_ERROR: all_executed_steps = dao.get_workflow_steps( executed_step.fk_workflow) for step in all_executed_steps: if step.step_index > executed_step.step_index: self.logger.debug( "Marking unreached operation %s with error." % step.fk_operation) unreached_operation = dao.get_operation_by_id( step.fk_operation) self.persist_operation_state( unreached_operation, model.STATUS_ERROR, "Blocked by failure in step %s with message: \n\n%s." % (executed_step.step_index, operation.additional_info)) workflow = dao.get_workflow_by_id(executed_step.fk_workflow) burst = dao.get_burst_by_id(workflow.fk_burst) self.mark_burst_finished(burst, error_message=operation.additional_info) dao.store_entity(burst) @staticmethod def _get_data(operation_id): """ For a given operation id, return the corresponding WorkflowStep and the NextWorkflowStep to be executed. """ executed_step = dao.get_workflow_step_for_operation(operation_id) if executed_step is not None: next_workflow_step = dao.get_workflow_step_by_step_index( executed_step.fk_workflow, executed_step.step_index + 1) return executed_step, next_workflow_step else: return None, None def mark_burst_finished(self, burst_entity, burst_status=None, error_message=None): """ Mark Burst status field. Also compute 'weight' for current burst: no of operations inside, estimate time on disk... :param burst_entity: BurstConfiguration to be updated, at finish time. :param burst_status: BurstConfiguration status. By default BURST_FINISHED :param error_message: If given, set the status to error and perpetuate the message. """ if burst_status is None: burst_status = model.BurstConfiguration.BURST_FINISHED if error_message is not None: burst_status = model.BurstConfiguration.BURST_ERROR try: ### If there are any DataType Groups in current Burst, update their counter. burst_dt_groups = dao.get_generic_entity(model.DataTypeGroup, burst_entity.id, "fk_parent_burst") for dt_group in burst_dt_groups: dt_group.count_results = dao.count_datatypes_in_group( dt_group.id) dt_group.disk_size, dt_group.subject = dao.get_summary_for_group( dt_group.id) dao.store_entity(dt_group) ### Update actual Burst entity fields burst_entity.datatypes_number = dao.count_datatypes_in_burst( burst_entity.id) burst_entity.workflows_number = dao.get_workflows_for_burst( burst_entity.id, is_count=True) burst_entity.status = burst_status burst_entity.error_message = error_message burst_entity.finish_time = datetime.now() dao.store_entity(burst_entity) except Exception: self.logger.exception( "Could not correctly update Burst status and meta-data!") burst_entity.status = burst_status burst_entity.error_message = "Error when updating Burst Status" burst_entity.finish_time = datetime.now() dao.store_entity(burst_entity)
class OperationService: """ Class responsible for preparing an operation launch. It will prepare parameters, and decide if the operation is to be executed immediately, or to be sent on the cluster. """ ATT_UID = "uid" def __init__(self): self.logger = get_logger(self.__class__.__module__) self.workflow_service = WorkflowService() self.file_helper = FilesHelper() ########################################################################################## ######## Methods related to launching operations start here ############################## ########################################################################################## def initiate_operation(self, current_user, project_id, adapter_instance, visible=True, **kwargs): """ Gets the parameters of the computation from the previous inputs form, and launches a computation (on the cluster or locally). Invoke custom method on an Adapter Instance. Make sure when the operation has finished that the correct results are stored into DB. """ if not isinstance(adapter_instance, ABCAdapter): self.logger.warning("Inconsistent Adapter Class:" + str(adapter_instance.__class__)) raise LaunchException("Developer Exception!!") ### Store Operation entity. algo = adapter_instance.stored_adapter algo_category = dao.get_category_by_id(algo.fk_category) operations = self.prepare_operations(current_user.id, project_id, algo, algo_category, {}, visible, **kwargs)[0] if isinstance(adapter_instance, ABCSynchronous): if len(operations) > 1: raise LaunchException( "Synchronous operations are not supporting ranges!") if len(operations) < 1: self.logger.warning("No operation was defined") raise LaunchException("Invalid empty Operation!!!") return self.initiate_prelaunch(operations[0], adapter_instance, **kwargs) else: return self._send_to_cluster(operations, adapter_instance, current_user.username) @staticmethod def _prepare_metadata(initial_metadata, algo_category, operation_group, submit_data): """ Gather metadata from submitted fields and current to be execute algorithm. Will populate STATE, GROUP in metadata """ metadata = copy(initial_metadata) user_group = None if DataTypeMetaData.KEY_OPERATION_TAG in submit_data: user_group = submit_data[DataTypeMetaData.KEY_OPERATION_TAG] if operation_group is not None: metadata[DataTypeMetaData.KEY_OPERATION_TAG] = operation_group.name if DataTypeMetaData.KEY_TAG_1 in submit_data: metadata[DataTypeMetaData.KEY_TAG_1] = submit_data[ DataTypeMetaData.KEY_TAG_1] metadata[DataTypeMetaData.KEY_STATE] = algo_category.defaultdatastate return metadata, user_group @staticmethod def _read_set(values): """ Parse a committed UI possible list of values, into a set converted into string.""" if isinstance(values, list): set_values = [] values_str = "" for val in values: if val not in set_values: set_values.append(val) values_str = values_str + " " + str(val) values = values_str return str(values).strip() def group_operation_launch(self, user_id, project_id, algorithm_id, category_id, existing_dt_group=None, **kwargs): """ Create and prepare the launch of a group of operations. """ category = dao.get_category_by_id(category_id) algorithm = dao.get_algorithm_by_id(algorithm_id) ops, _ = self.prepare_operations(user_id, project_id, algorithm, category, {}, existing_dt_group=existing_dt_group, **kwargs) for operation in ops: self.launch_operation(operation.id, True) def _prepare_metric_operation(self, sim_operation): # type: (Operation) -> None metric_algo = dao.get_algorithm_by_module( TimeseriesMetricsAdapter.__module__, TimeseriesMetricsAdapter.__name__) time_series_index = dao.get_generic_entity(TimeSeriesIndex, sim_operation.id, 'fk_from_operation')[0] ts_metrics_adapter_form = TimeseriesMetricsAdapterForm() ts_metrics_adapter_form.fill_from_trait( BaseTimeseriesMetricAlgorithm()) ts_metrics_adapter_form.time_series.data = time_series_index.gid op_params = json.dumps(ts_metrics_adapter_form.get_dict()) range_values = sim_operation.range_values metadata = { DataTypeMetaData.KEY_BURST: time_series_index.fk_parent_burst } metadata, user_group = self._prepare_metadata( metadata, metric_algo.algorithm_category, None, op_params) meta_str = json.dumps(metadata) parent_burst = dao.get_generic_entity( BurstConfiguration2, time_series_index.fk_parent_burst, 'id')[0] metric_operation_group_id = parent_burst.metric_operation_group_id metric_operation = Operation(sim_operation.fk_launched_by, sim_operation.fk_launched_in, metric_algo.id, op_params, meta_str, op_group_id=metric_operation_group_id, range_values=range_values) metric_operation.visible = False operation = dao.store_entity(metric_operation) metrics_datatype_group = dao.get_generic_entity( DataTypeGroup, metric_operation_group_id, 'fk_operation_group')[0] if metrics_datatype_group.fk_from_operation is None: metrics_datatype_group.fk_from_operation = metric_operation.id return operation def prepare_operations(self, user_id, project_id, algorithm, category, metadata, visible=True, existing_dt_group=None, **kwargs): """ Do all the necessary preparations for storing an operation. If it's the case of a range of values create an operation group and multiple operations for each possible instance from the range. :param metadata: Initial MetaData with potential Burst identification inside. """ operations = [] available_args, group = self._prepare_group(project_id, existing_dt_group, kwargs) if len(available_args) > TvbProfile.current.MAX_RANGE_NUMBER: raise LaunchException( "Too big range specified. You should limit the" " resulting operations to %d" % TvbProfile.current.MAX_RANGE_NUMBER) else: self.logger.debug("Launching a range with %d operations..." % len(available_args)) group_id = None if group is not None: group_id = group.id metadata, user_group = self._prepare_metadata(metadata, category, group, kwargs) self.logger.debug("Saving Operation(userId=" + str(user_id) + ",projectId=" + str(project_id) + "," + str(metadata) + ",algorithmId=" + str(algorithm.id) + ", ops_group= " + str(group_id) + ")") visible_operation = visible and category.display is False meta_str = json.dumps(metadata) for (one_set_of_args, range_vals) in available_args: range_values = json.dumps(range_vals) if range_vals else None operation = Operation(user_id, project_id, algorithm.id, json.dumps(one_set_of_args), meta_str, op_group_id=group_id, user_group=user_group, range_values=range_values) operation.visible = visible_operation operations.append(operation) operations = dao.store_entities(operations) if group is not None: burst_id = None if DataTypeMetaData.KEY_BURST in metadata: burst_id = metadata[DataTypeMetaData.KEY_BURST] if existing_dt_group is None: datatype_group = DataTypeGroup( group, operation_id=operations[0].id, fk_parent_burst=burst_id, state=metadata[DataTypeMetaData.KEY_STATE]) dao.store_entity(datatype_group) else: # Reset count existing_dt_group.count_results = None dao.store_entity(existing_dt_group) return operations, group def prepare_operations_for_workflowsteps(self, workflow_step_list, workflows, user_id, burst_id, project_id, group, sim_operations): """ Create and store Operation entities from a list of Workflow Steps. Will be generated workflows x workflow_step_list Operations. For every step in workflow_step_list one OperationGroup and one DataTypeGroup will be created (in case of PSE). """ for step in workflow_step_list: operation_group = None if (group is not None) and not isinstance(step, WorkflowStepView): operation_group = OperationGroup(project_id=project_id, ranges=group.range_references) operation_group = dao.store_entity(operation_group) operation = None metadata = {DataTypeMetaData.KEY_BURST: burst_id} algo_category = dao.get_algorithm_by_id(step.fk_algorithm) if algo_category is not None: algo_category = algo_category.algorithm_category for wf_idx, workflow in enumerate(workflows): cloned_w_step = step.clone() cloned_w_step.fk_workflow = workflow.id dynamic_params = cloned_w_step.dynamic_param op_params = cloned_w_step.static_param op_params.update(dynamic_params) range_values = None group_id = None if operation_group is not None: group_id = operation_group.id range_values = sim_operations[wf_idx].range_values if not isinstance(step, WorkflowStepView): ## For visualization steps, do not create operations, as those are not really needed. metadata, user_group = self._prepare_metadata( metadata, algo_category, operation_group, op_params) operation = Operation(user_id, project_id, step.fk_algorithm, json.dumps( op_params, cls=MapAsJson.MapAsJsonEncoder), meta=json.dumps(metadata), op_group_id=group_id, range_values=range_values, user_group=user_group) operation.visible = step.step_visible operation = dao.store_entity(operation) cloned_w_step.fk_operation = operation.id dao.store_entity(cloned_w_step) if operation_group is not None and operation is not None: datatype_group = DataTypeGroup( operation_group, operation_id=operation.id, fk_parent_burst=burst_id, state=metadata[DataTypeMetaData.KEY_STATE]) dao.store_entity(datatype_group) def initiate_prelaunch(self, operation, adapter_instance, **kwargs): """ Public method. This should be the common point in calling an adapter- method. """ result_msg = "" temp_files = [] try: unique_id = None if self.ATT_UID in kwargs: unique_id = kwargs[self.ATT_UID] #TODO: this currently keeps both ways to display forms if not 'SimulatorAdapter' in adapter_instance.__class__.__name__: if adapter_instance.get_input_tree() is None: filtered_kwargs = adapter_instance.get_form( ).get_form_values() else: filtered_kwargs = adapter_instance.prepare_ui_inputs( kwargs) params = dict() for k, value_ in filtered_kwargs.items(): params[str(k)] = value_ self.logger.debug("Launching operation " + str(operation.id) + " with " + str(filtered_kwargs)) else: params = kwargs self.logger.debug("Launching operation " + str(operation.id) + " with " + str(kwargs)) operation = dao.get_operation_by_id( operation.id) # Load Lazy fields disk_space_per_user = TvbProfile.current.MAX_DISK_SPACE pending_op_disk_space = dao.compute_disk_size_for_started_ops( operation.fk_launched_by) user_disk_space = dao.compute_user_generated_disk_size( operation.fk_launched_by) # From kB to Bytes available_space = disk_space_per_user - pending_op_disk_space - user_disk_space result_msg, nr_datatypes = adapter_instance._prelaunch( operation, unique_id, available_space, **params) operation = dao.get_operation_by_id(operation.id) ## Update DB stored kwargs for search purposes, to contain only valuable params (no unselected options) operation.parameters = json.dumps(kwargs) operation.mark_complete(STATUS_FINISHED) if nr_datatypes > 0: #### Write operation meta-XML only if some result are returned self.file_helper.write_operation_metadata(operation) dao.store_entity(operation) adapter_form = adapter_instance.get_form() try: temp_files = adapter_form.temporary_files except AttributeError: pass self._remove_files(temp_files) except zipfile.BadZipfile as excep: msg = "The uploaded file is not a valid ZIP!" self._handle_exception(excep, temp_files, msg, operation) except TVBException as excep: self._handle_exception(excep, temp_files, excep.message, operation) except MemoryError: msg = ( "Could not execute operation because there is not enough free memory." + " Please adjust operation parameters and re-launch it.") self._handle_exception(Exception(msg), temp_files, msg, operation) except Exception as excep1: msg = "Could not launch Operation with the given input data!" self._handle_exception(excep1, temp_files, msg, operation) if operation.fk_operation_group and 'SimulatorAdapter' in operation.algorithm.classname: next_op = self._prepare_metric_operation(operation) self.launch_operation(next_op.id) return result_msg def _send_to_cluster(self, operations, adapter_instance, current_username="******"): """ Initiate operation on cluster""" for operation in operations: try: BACKEND_CLIENT.execute(str(operation.id), current_username, adapter_instance) except Exception as excep: self._handle_exception(excep, {}, "Could not start operation!", operation) return operations def launch_operation(self, operation_id, send_to_cluster=False, adapter_instance=None): """ Method exposed for Burst-Workflow related calls. It is used for cascading operation in the same workflow. """ if operation_id is not None: operation = dao.get_operation_by_id(operation_id) if adapter_instance is None: algorithm = operation.algorithm adapter_instance = ABCAdapter.build_adapter(algorithm) parsed_params = utils.parse_json_parameters(operation.parameters) if not 'SimulatorAdapter' in adapter_instance.__class__.__name__: adapter_form = adapter_instance.get_form()() adapter_form.fill_from_post(parsed_params) adapter_instance.submit_form(adapter_form) if send_to_cluster: self._send_to_cluster([operation], adapter_instance, operation.user.username) else: self.initiate_prelaunch(operation, adapter_instance, **parsed_params) def _handle_exception(self, exception, temp_files, message, operation=None): """ Common way to treat exceptions: - remove temporary files, if any - set status ERROR on current operation (if any) - log exception """ self.logger.exception(message) if operation is not None: BurstService2().persist_operation_state(operation, STATUS_ERROR, str(exception)) self._remove_files(temp_files) exception.message = message raise exception.with_traceback( sys.exc_info()[2] ) # when rethrowing in python this is required to preserve the stack trace def _remove_files(self, file_list): """ Remove any files that exist in the file_dictionary. Currently used to delete temporary files created during an operation. """ for pth in file_list: pth = str(pth) try: if os.path.exists(pth) and os.path.isfile(pth): os.remove(pth) self.logger.debug("We no longer need file:" + pth + " => deleted") else: self.logger.warning("Trying to remove not existent file:" + pth) except OSError: self.logger.exception("Could not cleanup file!") @staticmethod def _range_name(range_no): return PARAM_RANGE_PREFIX + str(range_no) def _prepare_group(self, project_id, existing_dt_group, kwargs): """ Create and store OperationGroup entity, or return None """ # Standard ranges as accepted from UI range1_values = self.get_range_values(kwargs, self._range_name(1)) range2_values = self.get_range_values(kwargs, self._range_name(2)) available_args = self.__expand_arguments([(kwargs, None)], range1_values, self._range_name(1)) available_args = self.__expand_arguments(available_args, range2_values, self._range_name(2)) is_group = False ranges = [] if self._range_name(1) in kwargs and range1_values is not None: is_group = True ranges.append( json.dumps((kwargs[self._range_name(1)], range1_values))) if self._range_name(2) in kwargs and range2_values is not None: is_group = True ranges.append( json.dumps((kwargs[self._range_name(2)], range2_values))) # Now for additional ranges which might be the case for the 'model exploration' last_range_idx = 3 ranger_name = self._range_name(last_range_idx) while ranger_name in kwargs: values_for_range = self.get_range_values(kwargs, ranger_name) available_args = self.__expand_arguments(available_args, values_for_range, ranger_name) last_range_idx += 1 ranger_name = self._range_name(last_range_idx) if last_range_idx > 3: ranges = [ ] # Since we only have 3 fields in db for this just hide it if not is_group: group = None elif existing_dt_group is None: group = OperationGroup(project_id=project_id, ranges=ranges) group = dao.store_entity(group) else: group = existing_dt_group.parent_operation_group return available_args, group def get_range_values(self, kwargs, ranger_name): """ For the ranger given by ranger_name look in kwargs and return the array with all the possible values. """ if ranger_name not in kwargs: return None if str(kwargs[ranger_name]) not in kwargs: return None range_values = [] try: range_data = json.loads(str(kwargs[str(kwargs[ranger_name])])) except Exception: try: range_data = [ x.strip() for x in str(kwargs[str(kwargs[ranger_name])]).split(',') if len(x.strip()) > 0 ] return range_data except Exception: self.logger.exception("Could not launch operation !") raise LaunchException("Could not launch with no data from:" + str(ranger_name)) if type(range_data) in (list, tuple): return range_data if (constants.ATT_MINVALUE in range_data) and (constants.ATT_MAXVALUE in range_data): lo_val = float(range_data[constants.ATT_MINVALUE]) hi_val = float(range_data[constants.ATT_MAXVALUE]) step = float(range_data[constants.ATT_STEP]) range_values = list( Range(lo=lo_val, hi=hi_val, step=step).to_array()) #, mode=Range.MODE_INCLUDE_BOTH)) else: for possible_value in range_data: if range_data[possible_value]: range_values.append(possible_value) return range_values @staticmethod def __expand_arguments(arguments_list, range_values, range_title): """ Parse the arguments submitted from UI (flatten form) If any ranger is found, return a list of arguments for all possible operations. """ if range_values is None: return arguments_list result = [] for value in range_values: for args, range_ in arguments_list: kw_new = copy(args) range_new = copy(range_) kw_new[kw_new[range_title]] = value if range_new is None: range_new = {} range_new[kw_new[range_title]] = value del kw_new[range_title] result.append((kw_new, range_new)) return result ########################################################################################## ######## Methods related to stopping and restarting operations start here ################ ########################################################################################## def stop_operation(self, operation_id): """ Stop the operation given by the operation id. """ return BACKEND_CLIENT.stop_operation(int(operation_id))
class WorkflowService: """ service layer for work-flow entity. """ def __init__(self): self.logger = get_logger(self.__class__.__module__) self.file_helper = FilesHelper() def persist_operation_state(self, operation, operation_status, message=None): """ Update Operation instance state. Store it in DB and on HDD/ :param operation: Operation instance :param operation_status: new status :param message: message in case of error :return: operation instance changed """ operation.mark_complete(operation_status, unicode(message)) dao.store_entity(operation) operation = dao.get_operation_by_id(operation.id) self.file_helper.write_operation_metadata(operation) return operation @staticmethod def store_workflow_step(workflow_step): """ Store a workflow step entity. """ dao.store_entity(workflow_step) @staticmethod def create_and_store_workflow(project_id, burst_id, simulator_index, simulator_id, operations): """ Create and store the workflow given the project, user and burst in which the workflow is created. :param simulator_index: the index of the simulator in the workflow :param simulator_id: the id of the simulator adapter :param operations: a list with the operations created for the simulator steps """ workflows = [] for operation in operations: new_workflow = model.Workflow(project_id, burst_id) new_workflow = dao.store_entity(new_workflow) workflows.append(new_workflow) simulation_step = model.WorkflowStep( algorithm_id=simulator_id, workflow_id=new_workflow.id, step_index=simulator_index, static_param=operation.parameters) simulation_step.fk_operation = operation.id dao.store_entity(simulation_step) return workflows @staticmethod def set_dynamic_step_references(workflow_step, step_reference): """ :param workflow_step: a valid instance of a workflow_step :param step_reference: the step to which every dataType reference index should be set For each dynamic parameter of the given workflow_step set the 'step_index' at step_reference. """ dynamic_params = workflow_step.dynamic_param for entry in dynamic_params: dynamic_params[entry][ WorkflowStepConfiguration.STEP_INDEX_KEY] = step_reference workflow_step.dynamic_param = dynamic_params def prepare_next_step(self, last_executed_op_id): """ If the operation with id 'last_executed_op_id' resulted after the execution of a workflow step then this method will launch the operation corresponding to the next step from the workflow. """ try: current_step, next_workflow_step = self._get_data( last_executed_op_id) if next_workflow_step is not None: operation = dao.get_operation_by_id( next_workflow_step.fk_operation) dynamic_param_names = next_workflow_step.dynamic_workflow_param_names if len(dynamic_param_names) > 0: op_params = json.loads(operation.parameters) for param_name in dynamic_param_names: dynamic_param = op_params[param_name] former_step = dao.get_workflow_step_by_step_index( next_workflow_step.fk_workflow, dynamic_param[ WorkflowStepConfiguration.STEP_INDEX_KEY]) if type(dynamic_param[WorkflowStepConfiguration. DATATYPE_INDEX_KEY]) is IntType: datatypes = dao.get_results_for_operation( former_step.fk_operation) op_params[param_name] = datatypes[ dynamic_param[WorkflowStepConfiguration. DATATYPE_INDEX_KEY]].gid else: previous_operation = dao.get_operation_by_id( former_step.fk_operation) op_params[param_name] = json.loads( previous_operation.parameters)[ dynamic_param[WorkflowStepConfiguration. DATATYPE_INDEX_KEY]] operation.parameters = json.dumps(op_params) operation = dao.store_entity(operation) return operation.id elif current_step is not None: current_workflow = dao.get_workflow_by_id( current_step.fk_workflow) current_workflow.status = current_workflow.STATUS_FINISHED dao.store_entity(current_workflow) burst_entity = dao.get_burst_by_id(current_workflow.fk_burst) parallel_workflows = dao.get_workflows_for_burst( burst_entity.id) all_finished = True for workflow in parallel_workflows: if workflow.status == workflow.STATUS_STARTED: all_finished = False if all_finished: self.mark_burst_finished(burst_entity) return None except Exception, excep: self.logger.error(excep) self.logger.exception(excep) raise WorkflowInterStepsException(excep)
class ProjectService: """ Services layer for Project entities. """ def __init__(self): self.logger = get_logger(__name__) self.structure_helper = FilesHelper() def store_project(self, current_user, is_create, selected_id, **data): """ We want to create/update a project entity. """ # Validate Unique Name new_name = data["name"] if len(new_name) < 1: raise ProjectServiceException("Invalid project name!") projects_no = dao.count_projects_for_name(new_name, selected_id) if projects_no > 0: err = {'name': 'Please choose another name, this one is used!'} raise formencode.Invalid("Duplicate Name Error", {}, None, error_dict=err) started_operations = dao.get_operation_numbers(selected_id)[1] if started_operations > 0: raise ProjectServiceException("A project can not be renamed while operations are still running!") if is_create: current_proj = model.Project(new_name, current_user.id, data["description"]) self.structure_helper.get_project_folder(current_proj) else: try: current_proj = dao.get_project_by_id(selected_id) except Exception as excep: self.logger.exception("An error has occurred!") raise ProjectServiceException(str(excep)) if current_proj.name != new_name: self.structure_helper.rename_project_structure(current_proj.name, new_name) current_proj.name = new_name current_proj.description = data["description"] # Commit to make sure we have a valid ID current_proj.refresh_update_date() self.structure_helper.write_project_metadata(current_proj) current_proj = dao.store_entity(current_proj) # Retrieve, to initialize lazy attributes current_proj = dao.get_project_by_id(current_proj.id) # Update share settings on current Project entity visited_pages = [] prj_admin = current_proj.administrator.username if 'visited_pages' in data and data['visited_pages']: visited_pages = data['visited_pages'].split(',') for page in visited_pages: members = UserService.retrieve_all_users(prj_admin, int(page))[0] members = [m.id for m in members] dao.delete_members_for_project(current_proj.id, members) selected_user_ids = data["users"] dao.add_members_to_project(current_proj.id, selected_user_ids) # Finish operation self.logger.debug("Edit/Save OK for project:" + str(current_proj.id) + ' by user:'******'-' result["count"] = one_op[2] result["gid"] = one_op[13] if one_op[3] is not None and one_op[3]: try: operation_group = dao.get_generic_entity(model.OperationGroup, one_op[3])[0] result["group"] = operation_group.name result["group"] = result["group"].replace("_", " ") result["operation_group_id"] = operation_group.id datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3]) result["datatype_group_gid"] = datatype_group.gid result["gid"] = operation_group.gid ## Filter only viewers for current DataTypeGroup entity: result["view_groups"] = FlowService().get_visualizers_for_group(datatype_group.gid) except Exception: self.logger.exception("We will ignore group on entity:" + str(one_op)) result["datatype_group_gid"] = None else: result['group'] = None result['datatype_group_gid'] = None result["algorithm"] = dao.get_algorithm_by_id(one_op[4]) result["user"] = dao.get_user_by_id(one_op[5]) if type(one_op[6]) in (str, unicode): result["create"] = string2date(str(one_op[6])) else: result["create"] = one_op[6] if type(one_op[7]) in (str, unicode): result["start"] = string2date(str(one_op[7])) else: result["start"] = one_op[7] if type(one_op[8]) in (str, unicode): result["complete"] = string2date(str(one_op[8])) else: result["complete"] = one_op[8] if result["complete"] is not None and result["start"] is not None: result["duration"] = format_timedelta(result["complete"] - result["start"]) result["status"] = one_op[9] result["additional"] = one_op[10] result["visible"] = True if one_op[11] > 0 else False result['operation_tag'] = one_op[12] result['figures'] = None if not result['group']: datatype_results = dao.get_results_for_operation(result['id']) result['results'] = [] for dt in datatype_results: dt_loaded = ABCAdapter.load_entity_by_gid(dt.gid) if dt_loaded: result['results'].append(dt_loaded) else: self.logger.warn("Could not retrieve datatype %s" % str(dt)) operation_figures = dao.get_figures_for_operation(result['id']) # Compute the full path to the figure / image on disk for figure in operation_figures: figures_folder = self.structure_helper.get_images_folder(figure.project.name) figure_full_path = os.path.join(figures_folder, figure.file_path) # Compute the path available from browser figure.figure_path = utils.path2url_part(figure_full_path) result['figures'] = operation_figures else: result['results'] = None operations.append(result) except Exception: ## We got an exception when processing one Operation Row. We will continue with the rest of the rows. self.logger.exception("Could not prepare operation for display:" + str(one_op)) return selected_project, total_ops_nr, operations, pages_no def retrieve_projects_for_user(self, user_id, current_page=1): """ Return a list with all Projects visible for current user. """ start_idx = PROJECTS_PAGE_SIZE * (current_page - 1) total = dao.get_projects_for_user(user_id, is_count=True) available_projects = dao.get_projects_for_user(user_id, start_idx, PROJECTS_PAGE_SIZE) pages_no = total // PROJECTS_PAGE_SIZE + (1 if total % PROJECTS_PAGE_SIZE else 0) for prj in available_projects: fns, sta, err, canceled, pending = dao.get_operation_numbers(prj.id) prj.operations_finished = fns prj.operations_started = sta prj.operations_error = err prj.operations_canceled = canceled prj.operations_pending = pending prj.disk_size = dao.get_project_disk_size(prj.id) prj.disk_size_human = format_bytes_human(prj.disk_size) self.logger.debug("Displaying " + str(len(available_projects)) + " projects in UI for user " + str(user_id)) return available_projects, pages_no @staticmethod def get_linkable_projects_for_user(user_id, data_id): """ Find projects with are visible for current user, and in which current datatype hasn't been linked yet. """ return dao.get_linkable_projects_for_user(user_id, data_id) @transactional def remove_project(self, project_id): """ Remove Project from DB and File Storage. """ try: project2delete = dao.get_project_by_id(project_id) self.logger.debug("Deleting project: id=" + str(project_id) + ' name=' + project2delete.name) project_bursts = dao.get_bursts_for_project(project_id) for burst in project_bursts: dao.remove_entity(burst.__class__, burst.id) project_datatypes = dao.get_datatypes_in_project(project_id) for one_data in project_datatypes: self.remove_datatype(project_id, one_data.gid, True) links = dao.get_links_for_project(project_id) for one_link in links: dao.remove_entity(model.Links, one_link.id) self.structure_helper.remove_project_structure(project2delete.name) dao.delete_project(project_id) self.logger.debug("Deleted project: id=" + str(project_id) + ' name=' + project2delete.name) except RemoveDataTypeException as excep: self.logger.exception("Could not execute operation Node Remove!") raise ProjectServiceException(str(excep)) except FileStructureException as excep: self.logger.exception("Could not delete because of rights!") raise ProjectServiceException(str(excep)) except Exception as excep: self.logger.exception(str(excep)) raise ProjectServiceException(str(excep)) # ----------------- Methods for populating Data-Structure Page --------------- @staticmethod def get_datatype_in_group(group): """ Return all dataTypes that are the result of the same DTgroup. """ return dao.get_datatype_in_group(datatype_group_id=group) @staticmethod def get_datatypes_from_datatype_group(datatype_group_id): """ Retrieve all dataType which are part from the given dataType group. """ return dao.get_datatypes_from_datatype_group(datatype_group_id) @staticmethod def load_operation_by_gid(operation_gid): """ Retrieve loaded Operation from DB""" return dao.get_operation_by_gid(operation_gid) @staticmethod def get_operation_group_by_id(operation_group_id): """ Loads OperationGroup from DB""" return dao.get_operationgroup_by_id(operation_group_id) @staticmethod def get_operation_group_by_gid(operation_group_gid): """ Loads OperationGroup from DB""" return dao.get_operationgroup_by_gid(operation_group_gid) @staticmethod def get_operations_in_group(operation_group): """ Return all the operations from an operation group. """ return dao.get_operations_in_group(operation_group.id) @staticmethod def is_upload_operation(operation_gid): """ Returns True only if the operation with the given GID is an upload operation. """ return dao.is_upload_operation(operation_gid) @staticmethod def get_all_operations_for_uploaders(project_id): """ Returns all finished upload operations. """ return dao.get_all_operations_for_uploaders(project_id) def set_operation_and_group_visibility(self, entity_gid, is_visible, is_operation_group=False): """ Sets the operation visibility. If 'is_operation_group' is True than this method will change the visibility for all the operation from the OperationGroup with the GID field equal to 'entity_gid'. """ def set_visibility(op): # workaround: # 'reload' the operation so that it has the project property set. # get_operations_in_group does not eager load it and now we're out of a sqlalchemy session # write_operation_metadata requires that property op = dao.get_operation_by_id(op.id) # end hack op.visible = is_visible self.structure_helper.write_operation_metadata(op) dao.store_entity(op) def set_group_descendants_visibility(operation_group_id): ops_in_group = dao.get_operations_in_group(operation_group_id) for group_op in ops_in_group: set_visibility(group_op) if is_operation_group: op_group_id = dao.get_operationgroup_by_gid(entity_gid).id set_group_descendants_visibility(op_group_id) else: operation = dao.get_operation_by_gid(entity_gid) # we assure that if the operation belongs to a group than the visibility will be changed for the entire group if operation.fk_operation_group is not None: set_group_descendants_visibility(operation.fk_operation_group) else: set_visibility(operation) def get_operation_details(self, operation_gid, is_group): """ :returns: an entity OperationOverlayDetails filled with all information for current operation details. """ if is_group: operation_group = self.get_operation_group_by_gid(operation_gid) operation = dao.get_operations_in_group(operation_group.id, False, True) ## Reload, to make sure all attributes lazy are populated as well. operation = dao.get_operation_by_gid(operation.gid) no_of_op_in_group = dao.get_operations_in_group(operation_group.id, is_count=True) datatype_group = self.get_datatypegroup_by_op_group_id(operation_group.id) count_result = dao.count_datatypes_in_group(datatype_group.id) else: operation = dao.get_operation_by_gid(operation_gid) if operation is None: return None no_of_op_in_group = 1 count_result = dao.count_resulted_datatypes(operation.id) username = dao.get_user_by_id(operation.fk_launched_by).username burst = dao.get_burst_for_operation_id(operation.id) datatypes_param, all_special_params = self._review_operation_inputs(operation.gid) op_pid = dao.get_operation_process_for_operation(operation.id) op_details = OperationOverlayDetails(operation, username, len(datatypes_param), count_result, burst, no_of_op_in_group, op_pid) ## Add all parameter which are set differently by the user on this Operation. if all_special_params is not None: op_details.add_scientific_fields(all_special_params) return op_details @staticmethod def get_filterable_meta(): """ Contains all the attributes by which the user can structure the tree of DataTypes """ return DataTypeMetaData.get_filterable_meta() def get_project_structure(self, project, visibility_filter, first_level, second_level, filter_value): """ Find all DataTypes (including the linked ones and the groups) relevant for the current project. In case of a problem, will return an empty list. """ metadata_list = [] dt_list = dao.get_data_in_project(project.id, visibility_filter, filter_value) for dt in dt_list: # Prepare the DT results from DB, for usage in controller, by converting into DataTypeMetaData objects data = {} is_group = False group_op = None dt_entity = dao.get_datatype_by_gid(dt.gid) if dt_entity is None: self.logger.warning("Ignored entity (possibly removed DT class)" + str(dt)) continue ## Filter by dt.type, otherwise Links to individual DT inside a group will be mistaken if dt.type == "DataTypeGroup" and dt.parent_operation.operation_group is not None: is_group = True group_op = dt.parent_operation.operation_group # All these fields are necessary here for dynamic Tree levels. data[DataTypeMetaData.KEY_DATATYPE_ID] = dt.id data[DataTypeMetaData.KEY_GID] = dt.gid data[DataTypeMetaData.KEY_NODE_TYPE] = dt.type data[DataTypeMetaData.KEY_STATE] = dt.state data[DataTypeMetaData.KEY_SUBJECT] = str(dt.subject) data[DataTypeMetaData.KEY_TITLE] = dt_entity.display_name data[DataTypeMetaData.KEY_RELEVANCY] = dt.visible data[DataTypeMetaData.KEY_LINK] = dt.parent_operation.fk_launched_in != project.id data[DataTypeMetaData.KEY_TAG_1] = dt.user_tag_1 if dt.user_tag_1 else '' data[DataTypeMetaData.KEY_TAG_2] = dt.user_tag_2 if dt.user_tag_2 else '' data[DataTypeMetaData.KEY_TAG_3] = dt.user_tag_3 if dt.user_tag_3 else '' data[DataTypeMetaData.KEY_TAG_4] = dt.user_tag_4 if dt.user_tag_4 else '' data[DataTypeMetaData.KEY_TAG_5] = dt.user_tag_5 if dt.user_tag_5 else '' # Operation related fields: operation_name = CommonDetails.compute_operation_name( dt.parent_operation.algorithm.algorithm_category.displayname, dt.parent_operation.algorithm.displayname) data[DataTypeMetaData.KEY_OPERATION_TYPE] = operation_name data[DataTypeMetaData.KEY_OPERATION_ALGORITHM] = dt.parent_operation.algorithm.displayname data[DataTypeMetaData.KEY_AUTHOR] = dt.parent_operation.user.username data[DataTypeMetaData.KEY_OPERATION_TAG] = group_op.name if is_group else dt.parent_operation.user_group data[DataTypeMetaData.KEY_OP_GROUP_ID] = group_op.id if is_group else None completion_date = dt.parent_operation.completion_date string_year = completion_date.strftime(MONTH_YEAR_FORMAT) if completion_date is not None else "" string_month = completion_date.strftime(DAY_MONTH_YEAR_FORMAT) if completion_date is not None else "" data[DataTypeMetaData.KEY_DATE] = date2string(completion_date) if (completion_date is not None) else '' data[DataTypeMetaData.KEY_CREATE_DATA_MONTH] = string_year data[DataTypeMetaData.KEY_CREATE_DATA_DAY] = string_month data[DataTypeMetaData.KEY_BURST] = dt._parent_burst.name if dt._parent_burst is not None else '-None-' metadata_list.append(DataTypeMetaData(data, dt.invalid)) return StructureNode.metadata2tree(metadata_list, first_level, second_level, project.id, project.name) @staticmethod def get_datatype_details(datatype_gid): """ :returns: an array. First entry in array is an instance of DataTypeOverlayDetails\ The second one contains all the possible states for the specified dataType. """ meta_atts = DataTypeOverlayDetails() states = DataTypeMetaData.STATES try: datatype_result = dao.get_datatype_details(datatype_gid) meta_atts.fill_from_datatype(datatype_result, datatype_result._parent_burst) return meta_atts, states, datatype_result except Exception: ## We ignore exception here (it was logged above, and we want to return no details). return meta_atts, states, None def _remove_project_node_files(self, project_id, gid, skip_validation=False): """ Delegate removal of a node in the structure of the project. In case of a problem will THROW StructureException. """ try: project = self.find_project(project_id) datatype = dao.get_datatype_by_gid(gid) links = dao.get_links_for_datatype(datatype.id) if links: was_link = False for link in links: # This means it's only a link and we need to remove it if link.fk_from_datatype == datatype.id and link.fk_to_project == project.id: dao.remove_entity(model.Links, link.id) was_link = True if not was_link: # Create a clone of the operation new_op = model.Operation(dao.get_system_user().id, links[0].fk_to_project, datatype.parent_operation.fk_from_algo, datatype.parent_operation.parameters, datatype.parent_operation.meta_data, datatype.parent_operation.status, datatype.parent_operation.start_date, datatype.parent_operation.completion_date, datatype.parent_operation.fk_operation_group, datatype.parent_operation.additional_info, datatype.parent_operation.user_group, datatype.parent_operation.range_values) new_op = dao.store_entity(new_op) to_project = self.find_project(links[0].fk_to_project).name new_op_loaded = dao.get_operation_by_id(new_op.id) self.structure_helper.write_operation_metadata(new_op_loaded) self.structure_helper.move_datatype(datatype, to_project, str(new_op.id)) datatype.set_operation_id(new_op.id) datatype.parent_operation = new_op dao.store_entity(datatype) dao.remove_entity(model.Links, links[0].id) else: specific_remover = get_remover(datatype.type)(datatype) specific_remover.remove_datatype(skip_validation) self.structure_helper.remove_datatype(datatype) except RemoveDataTypeException: self.logger.exception("Could not execute operation Node Remove!") raise except FileStructureException: self.logger.exception("Remove operation failed") raise StructureException("Remove operation failed for unknown reasons.Please contact system administrator.") def remove_operation(self, operation_id): """ Remove a given operation """ operation = dao.try_get_operation_by_id(operation_id) if operation is not None: self.logger.debug("Deleting operation %s " % operation) datatypes_for_op = dao.get_results_for_operation(operation_id) for dt in reversed(datatypes_for_op): self.remove_datatype(operation.project.id, dt.gid, False) dao.remove_entity(model.Operation, operation.id) self.logger.debug("Finished deleting operation %s " % operation) else: self.logger.warning("Attempt to delete operation with id=%s which no longer exists." % operation_id) def remove_datatype(self, project_id, datatype_gid, skip_validation=False): """ Method used for removing a dataType. If the given dataType is a DatatypeGroup or a dataType from a DataTypeGroup than this method will remove the entire group. The operation(s) used for creating the dataType(s) will also be removed. """ datatype = dao.get_datatype_by_gid(datatype_gid) if datatype is None: self.logger.warning("Attempt to delete DT[%s] which no longer exists." % datatype_gid) return is_datatype_group = False if dao.is_datatype_group(datatype_gid): is_datatype_group = True elif datatype.fk_datatype_group is not None: is_datatype_group = True datatype = dao.get_datatype_by_id(datatype.fk_datatype_group) operations_set = [datatype.fk_from_operation] correct = True if is_datatype_group: self.logger.debug("Removing datatype group %s" % datatype) data_list = dao.get_datatypes_from_datatype_group(datatype.id) for adata in data_list: self._remove_project_node_files(project_id, adata.gid, skip_validation) if adata.fk_from_operation not in operations_set: operations_set.append(adata.fk_from_operation) datatype_group = dao.get_datatype_group_by_gid(datatype.gid) dao.remove_datatype(datatype_gid) correct = correct and dao.remove_entity(model.OperationGroup, datatype_group.fk_operation_group) else: self.logger.debug("Removing datatype %s" % datatype) self._remove_project_node_files(project_id, datatype.gid, skip_validation) ## Remove Operation entity in case no other DataType needs them. project = dao.get_project_by_id(project_id) for operation_id in operations_set: dependent_dt = dao.get_generic_entity(model.DataType, operation_id, "fk_from_operation") if len(dependent_dt) > 0: ### Do not remove Operation in case DataType still exist referring it. continue correct = correct and dao.remove_entity(model.Operation, operation_id) ## Make sure Operation folder is removed self.structure_helper.remove_operation_data(project.name, datatype.fk_from_operation) if not correct: raise RemoveDataTypeException("Could not remove DataType " + str(datatype_gid)) def update_metadata(self, submit_data): """ Update DataType/ DataTypeGroup metadata THROW StructureException when input data is invalid. """ new_data = dict() for key in DataTypeOverlayDetails().meta_attributes_list: if key in submit_data: new_data[key] = submit_data[key] if new_data[CommonDetails.CODE_OPERATION_TAG] == '': new_data[CommonDetails.CODE_OPERATION_TAG] = None try: if (CommonDetails.CODE_OPERATION_GROUP_ID in new_data and new_data[CommonDetails.CODE_OPERATION_GROUP_ID] and new_data[CommonDetails.CODE_OPERATION_GROUP_ID] != ''): # We need to edit a group all_data_in_group = dao.get_datatype_in_group(operation_group_id= new_data[CommonDetails.CODE_OPERATION_GROUP_ID]) if len(all_data_in_group) < 1: raise StructureException("Inconsistent group, can not be updated!") datatype_group = dao.get_generic_entity(model.DataTypeGroup, all_data_in_group[0].fk_datatype_group)[0] all_data_in_group.append(datatype_group) for datatype in all_data_in_group: new_data[CommonDetails.CODE_GID] = datatype.gid self._edit_data(datatype, new_data, True) else: # Get the required DataType and operation from DB to store changes that will be done in XML. gid = new_data[CommonDetails.CODE_GID] datatype = dao.get_datatype_by_gid(gid) self._edit_data(datatype, new_data) except Exception as excep: self.logger.exception(excep) raise StructureException(str(excep)) def _edit_data(self, datatype, new_data, from_group=False): """ Private method, used for editing a meta-data XML file and a DataType row for a given custom DataType entity with new dictionary of data from UI. """ if isinstance(datatype, MappedType) and not os.path.exists(datatype.get_storage_file_path()): if not datatype.invalid: datatype.invalid = True dao.store_entity(datatype) return # 1. First update Operation fields: # Update group field if possible new_group_name = new_data[CommonDetails.CODE_OPERATION_TAG] empty_group_value = (new_group_name is None or new_group_name == "") if from_group: if empty_group_value: raise StructureException("Empty group is not allowed!") group = dao.get_generic_entity(model.OperationGroup, new_data[CommonDetails.CODE_OPERATION_GROUP_ID]) if group and len(group) > 0 and new_group_name != group[0].name: group = group[0] exists_group = dao.get_generic_entity(model.OperationGroup, new_group_name, 'name') if exists_group: raise StructureException("Group '" + new_group_name + "' already exists.") group.name = new_group_name dao.store_entity(group) else: operation = dao.get_operation_by_id(datatype.fk_from_operation) operation.user_group = new_group_name dao.store_entity(operation) # 2. Update dateType fields: datatype.subject = new_data[DataTypeOverlayDetails.DATA_SUBJECT] datatype.state = new_data[DataTypeOverlayDetails.DATA_STATE] if DataTypeOverlayDetails.DATA_TAG_1 in new_data: datatype.user_tag_1 = new_data[DataTypeOverlayDetails.DATA_TAG_1] if DataTypeOverlayDetails.DATA_TAG_2 in new_data: datatype.user_tag_2 = new_data[DataTypeOverlayDetails.DATA_TAG_2] if DataTypeOverlayDetails.DATA_TAG_3 in new_data: datatype.user_tag_3 = new_data[DataTypeOverlayDetails.DATA_TAG_3] if DataTypeOverlayDetails.DATA_TAG_4 in new_data: datatype.user_tag_4 = new_data[DataTypeOverlayDetails.DATA_TAG_4] if DataTypeOverlayDetails.DATA_TAG_5 in new_data: datatype.user_tag_5 = new_data[DataTypeOverlayDetails.DATA_TAG_5] datatype = dao.store_entity(datatype) # 3. Update MetaData in H5 as well. datatype.persist_full_metadata() # 4. Update the group_name/user_group into the operation meta-data file operation = dao.get_operation_by_id(datatype.fk_from_operation) self.structure_helper.update_operation_metadata(operation.project.name, new_group_name, str(datatype.fk_from_operation), from_group) def get_datatype_and_datatypegroup_inputs_for_operation(self, operation_gid, selected_filter): """ Returns the dataTypes that are used as input parameters for the given operation. 'selected_filter' - is expected to be a visibility filter. If any dataType is part of a dataType group then the dataType group will be returned instead of that dataType. """ all_datatypes = self._review_operation_inputs(operation_gid)[0] datatype_inputs = [] for datatype in all_datatypes: if selected_filter.display_name == StaticFiltersFactory.RELEVANT_VIEW: if datatype.visible: datatype_inputs.append(datatype) else: datatype_inputs.append(datatype) datatypes = [] datatype_groups = dict() for data_type in datatype_inputs: if data_type.fk_datatype_group is None: datatypes.append(data_type) elif data_type.fk_datatype_group not in datatype_groups: dt_group = dao.get_datatype_by_id(data_type.fk_datatype_group) datatype_groups[data_type.fk_datatype_group] = dt_group datatypes.extend([v for _, v in six.iteritems(datatype_groups)]) return datatypes def _review_operation_inputs(self, operation_gid): """ :returns: A list of DataTypes that are used as input parameters for the specified operation. And a dictionary will all operation parameters different then the default ones. """ operation = dao.get_operation_by_gid(operation_gid) parameters = json.loads(operation.parameters) try: adapter = ABCAdapter.build_adapter(operation.algorithm) return adapter.review_operation_inputs(parameters) except IntrospectionException: self.logger.warning("Could not find adapter class for operation %s" % operation_gid) inputs_datatypes = [] changed_parameters = dict(Warning="Algorithm was Removed. We can not offer more details") for submit_param in parameters.values(): self.logger.debug("Searching DT by GID %s" % submit_param) datatype = ABCAdapter.load_entity_by_gid(str(submit_param)) if datatype is not None: inputs_datatypes.append(datatype) return inputs_datatypes, changed_parameters def get_datatypes_inputs_for_operation_group(self, group_id, selected_filter): """ Returns the dataType inputs for an operation group. If more dataTypes are part of the same dataType group then only the dataType group will be returned instead of them. """ operations_gids = dao.get_operations_in_group(group_id, only_gids=True) op_group_inputs = dict() for gid in operations_gids: op_inputs = self.get_datatype_and_datatypegroup_inputs_for_operation(gid[0], selected_filter) for datatype in op_inputs: op_group_inputs[datatype.id] = datatype return op_group_inputs.values() @staticmethod def get_results_for_operation(operation_id, selected_filter=None): """ Retrieve the DataTypes entities resulted after the execution of the given operation. """ return dao.get_results_for_operation(operation_id, selected_filter) @staticmethod def get_operations_for_datatype_group(datatype_group_id, visibility_filter, only_in_groups=False): """ Returns all the operations which uses as an input parameter a dataType from the given DataTypeGroup. visibility_filter - is a filter used for retrieving all the operations or only the relevant ones. If only_in_groups is True than this method will return only the operations that are part from an operation group, otherwise it will return only the operations that are NOT part of an operation group. """ if visibility_filter.display_name != StaticFiltersFactory.RELEVANT_VIEW: return dao.get_operations_for_datatype_group(datatype_group_id, only_relevant=False, only_in_groups=only_in_groups) return dao.get_operations_for_datatype_group(datatype_group_id, only_in_groups=only_in_groups) @staticmethod def get_operations_for_datatype(datatype_gid, visibility_filter, only_in_groups=False): """ Returns all the operations which uses as an input parameter the dataType with the specified GID. If only_in_groups is True than this method will return only the operations that are part from an operation group, otherwise it will return only the operations that are NOT part of an operation group. """ if visibility_filter.display_name != StaticFiltersFactory.RELEVANT_VIEW: return dao.get_operations_for_datatype(datatype_gid, only_relevant=False, only_in_groups=only_in_groups) return dao.get_operations_for_datatype(datatype_gid, only_in_groups=only_in_groups) @staticmethod def get_datatype_by_id(datatype_id): """Retrieve a DataType DB reference by its id.""" return dao.get_datatype_by_id(datatype_id) @staticmethod def get_datatypegroup_by_gid(datatypegroup_gid): """ Returns the DataTypeGroup with the specified gid. """ return dao.get_datatype_group_by_gid(datatypegroup_gid) @staticmethod def count_datatypes_generated_from(datatype_gid): """ A list with all the datatypes resulted from operations that had as input the datatype given by 'datatype_gid'. """ return dao.count_datatypes_generated_from(datatype_gid) @staticmethod def get_datatypegroup_by_op_group_id(operation_group_id): """ Returns the DataTypeGroup with the specified id. """ return dao.get_datatypegroup_by_op_group_id(operation_group_id) @staticmethod def set_datatype_visibility(datatype_gid, is_visible): """ Sets the dataType visibility. If the given dataType is a dataType group or it is part of a dataType group than this method will set the visibility for each dataType from this group. """ def set_visibility(dt): """ set visibility flag, persist in db and h5""" dt.visible = is_visible dt = dao.store_entity(dt) dt.persist_full_metadata() def set_group_descendants_visibility(datatype_group_id): datatypes_in_group = dao.get_datatypes_from_datatype_group(datatype_group_id) for group_dt in datatypes_in_group: set_visibility(group_dt) datatype = dao.get_datatype_by_gid(datatype_gid) if isinstance(datatype, DataTypeGroup): # datatype is a group set_group_descendants_visibility(datatype.id) elif datatype.fk_datatype_group is not None: # datatype is member of a group set_group_descendants_visibility(datatype.fk_datatype_group) # the datatype to be updated is the parent datatype group datatype = dao.get_datatype_by_id(datatype.fk_datatype_group) # update the datatype or datatype group. set_visibility(datatype) @staticmethod def is_datatype_group(datatype_gid): """ Used to check if the dataType with the specified GID is a DataTypeGroup. """ return dao.is_datatype_group(datatype_gid)
def _adapt_simulation_monitor_params(): """ For previous simulation with EEG monitor, adjust the change of input parameters. """ session = SA_SESSIONMAKER() param_connectivity = "connectivity" param_eeg_proj_old = "monitors_parameters_option_EEG_projection_matrix_data" param_eeg_proj_new = "monitors_parameters_option_EEG_projection" param_eeg_sensors = "monitors_parameters_option_EEG_sensors" param_eeg_rm = "monitors_parameters_option_EEG_region_mapping" try: all_eeg_ops = session.query(model.Operation).filter( model.Operation.parameters.ilike('%"' + param_eeg_proj_old + '"%')).all() files_helper = FilesHelper() all_bursts = dict() for eeg_op in all_eeg_ops: try: op_params = parse_json_parameters(eeg_op.parameters) LOGGER.debug("Updating " + str(op_params)) old_projection_guid = op_params[param_eeg_proj_old] connectivity_guid = op_params[param_connectivity] rm = dao.get_generic_entity(RegionMapping, connectivity_guid, "_connectivity")[0] dt = dao.get_generic_entity(model.DataType, old_projection_guid, "gid")[0] if dt.type == 'ProjectionSurfaceEEG': LOGGER.debug("Previous Prj is surface: " + old_projection_guid) new_projection_guid = old_projection_guid else: new_projection_guid = session.execute(text("""SELECT DT.gid FROM "MAPPED_PROJECTION_MATRIX_DATA" PMO, "DATA_TYPES" DTO, "MAPPED_PROJECTION_MATRIX_DATA" PM, "DATA_TYPES" DT WHERE DTO.id=PMO.id and DT.id=PM.id and PM._sensors=PMO._sensors and PM._sources='""" + rm._surface + """' and DTO.gid='""" + old_projection_guid + """';""")).fetchall()[0][0] LOGGER.debug("New Prj is surface: " + str(new_projection_guid)) sensors_guid = session.execute(text("""SELECT _sensors FROM "MAPPED_PROJECTION_MATRIX_DATA" WHERE id = '""" + str(dt.id) + """';""")).fetchall()[0][0] del op_params[param_eeg_proj_old] op_params[param_eeg_proj_new] = str(new_projection_guid) op_params[param_eeg_sensors] = str(sensors_guid) op_params[param_eeg_rm] = str(rm.gid) eeg_op.parameters = json.dumps(op_params, cls=MapAsJson.MapAsJsonEncoder) LOGGER.debug("New params:" + eeg_op.parameters) files_helper.write_operation_metadata(eeg_op) burst = dao.get_burst_for_operation_id(eeg_op.id) if burst is not None: LOGGER.debug("Updating burst:" + str(burst)) burst.prepare_after_load() del burst.simulator_configuration[param_eeg_proj_old] burst.simulator_configuration[param_eeg_proj_new] = {'value': str(new_projection_guid)} burst.simulator_configuration[param_eeg_sensors] = {'value': str(sensors_guid)} burst.simulator_configuration[param_eeg_rm] = {'value': str(rm.gid)} burst._simulator_configuration = json.dumps(burst.simulator_configuration, cls=MapAsJson.MapAsJsonEncoder) if burst.id not in all_bursts: all_bursts[burst.id] = burst except Exception: LOGGER.exception("Could not process " + str(eeg_op)) session.add_all(all_eeg_ops) session.add_all(list(all_bursts.values())) session.commit() except Exception: LOGGER.exception("Could not update Simulation Monitor Params") finally: session.close()
class BurstService(object): LAUNCH_NEW = 'new' LAUNCH_BRANCH = 'branch' def __init__(self): self.logger = get_logger(self.__class__.__module__) self.file_helper = FilesHelper() def mark_burst_finished(self, burst_entity, burst_status=None, error_message=None): """ Mark Burst status field. Also compute 'weight' for current burst: no of operations inside, estimate time on disk... :param burst_entity: BurstConfiguration to be updated, at finish time. :param burst_status: BurstConfiguration status. By default BURST_FINISHED :param error_message: If given, set the status to error and perpetuate the message. """ if burst_status is None: burst_status = BurstConfiguration.BURST_FINISHED if error_message is not None: burst_status = BurstConfiguration.BURST_ERROR try: # If there are any DataType Groups in current Burst, update their counter. burst_dt_groups = dao.get_generic_entity(DataTypeGroup, burst_entity.id, "fk_parent_burst") for dt_group in burst_dt_groups: dt_group.count_results = dao.count_datatypes_in_group( dt_group.id) dt_group.disk_size, dt_group.subject = dao.get_summary_for_group( dt_group.id) dao.store_entity(dt_group) # Update actual Burst entity fields burst_entity.datatypes_number = dao.count_datatypes_in_burst( burst_entity.id) burst_entity.status = burst_status burst_entity.error_message = error_message burst_entity.finish_time = datetime.now() dao.store_entity(burst_entity) self.update_burst_configuration_h5(burst_entity) except Exception: self.logger.exception( "Could not correctly update Burst status and meta-data!") burst_entity.status = burst_status burst_entity.error_message = "Error when updating Burst Status" burst_entity.finish_time = datetime.now() dao.store_entity(burst_entity) self.update_burst_configuration_h5(burst_entity) def persist_operation_state(self, operation, operation_status, message=None): """ Update Operation instance state. Store it in DB and on HDD/ :param operation: Operation instance :param operation_status: new status :param message: message in case of error :return: operation instance changed """ operation.mark_complete(operation_status, message) dao.store_entity(operation) operation = dao.get_operation_by_id(operation.id) self.file_helper.write_operation_metadata(operation) # update burst also burst_config = self.get_burst_for_operation_id(operation.id) if burst_config is not None: burst_status = STATUS_FOR_OPERATION.get(operation_status) self.mark_burst_finished(burst_config, burst_status, message) return operation def get_burst_for_operation_id(self, operation_id): return dao.get_burst_for_operation_id(operation_id) def rename_burst(self, burst_id, new_name): """ Rename the burst given by burst_id, setting it's new name to burst_name. """ burst = dao.get_burst_by_id(burst_id) burst.name = new_name dao.store_entity(burst) self.update_burst_configuration_h5(burst) @staticmethod def get_available_bursts(project_id): """ Return all the burst for the current project. """ bursts = dao.get_bursts_for_project( project_id, page_size=MAX_BURSTS_DISPLAYED) or [] # for burst in bursts: # burst.prepare_after_load() return bursts @staticmethod def populate_burst_disk_usage(bursts): """ Adds a disk_usage field to each burst object. The disk usage is computed as the sum of the datatypes generated by a burst """ sizes = dao.compute_bursts_disk_size([b.id for b in bursts]) for b in bursts: b.disk_size = format_bytes_human(sizes[b.id]) def update_history_status(self, id_list): """ For each burst_id received in the id_list read new status from DB and return a list [id, new_status] pair. """ result = [] for b_id in id_list: burst = dao.get_burst_by_id(b_id) # burst.prepare_after_load() if burst is not None: if burst.status == burst.BURST_RUNNING: running_time = datetime.now() - burst.start_time else: running_time = burst.finish_time - burst.start_time running_time = format_timedelta(running_time, most_significant2=False) if burst.status == burst.BURST_ERROR: msg = 'Check Operations page for error Message' else: msg = '' result.append([ burst.id, burst.status, burst.is_group, msg, running_time ]) else: self.logger.debug("Could not find burst with id=" + str(b_id) + ". Might have been deleted by user!!") return result # TODO: We should implement these two methods def stop_burst(self, burst): raise NotImplementedError def cancel_or_remove_burst(self, burst_id): raise NotImplementedError @staticmethod def update_simulation_fields(burst_id, op_simulation_id, simulation_gid): burst = dao.get_burst_by_id(burst_id) burst.fk_simulation = op_simulation_id burst.simulator_gid = simulation_gid.hex burst = dao.store_entity(burst) return burst def update_burst_configuration_h5(self, burst_configuration): # type: (BurstConfiguration) -> None project = dao.get_project_by_id(burst_configuration.fk_project) storage_path = self.file_helper.get_project_folder( project, str(burst_configuration.fk_simulation)) self.store_burst_configuration(burst_configuration, storage_path) def load_burst_configuration(self, burst_config_id): # type: (int) -> BurstConfiguration burst_config = dao.get_burst_by_id(burst_config_id) return burst_config def prepare_burst_for_pse(self, burst_config): # type: (BurstConfiguration) -> None if burst_config.range2: ranges = [burst_config.range1, burst_config.range2] else: ranges = [burst_config.range1] operation_group = OperationGroup(burst_config.fk_project, ranges=ranges) operation_group = dao.store_entity(operation_group) metric_operation_group = OperationGroup(burst_config.fk_project, ranges=ranges) metric_operation_group = dao.store_entity(metric_operation_group) burst_config.operation_group = operation_group burst_config.fk_operation_group = operation_group.id burst_config.metric_operation_group = metric_operation_group burst_config.fk_metric_operation_group = metric_operation_group.id dao.store_entity(burst_config) def store_burst_configuration(self, burst_config, storage_path): bc_path = h5.path_for(storage_path, BurstConfigurationH5, burst_config.gid) with BurstConfigurationH5(bc_path) as bc_h5: bc_h5.store(burst_config) def load_burst_configuration_from_folder(self, simulator_folder, project): bc_h5_filename = DirLoader( simulator_folder, None).find_file_for_has_traits_type(BurstConfiguration) burst_config = BurstConfiguration(project.id) with BurstConfigurationH5( os.path.join(simulator_folder, bc_h5_filename)) as bc_h5: bc_h5.load_into(burst_config) return burst_config @staticmethod def prepare_name(burst, project_id): simulation_number = dao.get_number_of_bursts(project_id) + 1 if burst.name is None: default_simulation_name = 'simulation_' + str(simulation_number) else: default_simulation_name = burst.name return default_simulation_name, simulation_number