def _export_linked_datatypes(self, project, zip_file): files_helper = FilesHelper() linked_paths = self._get_linked_datatypes_storage_path(project) if not linked_paths: # do not export an empty operation return # Make a import operation which will contain links to other projects alg_group = dao.find_group(TVB_IMPORTER_MODULE, TVB_IMPORTER_CLASS) algo = dao.get_algorithm_by_group(alg_group.id) op = model.Operation(None, project.id, algo.id, '') op.project = project op.algorithm = algo op.id = 'links-to-external-projects' op.start_now() op.mark_complete(model.STATUS_FINISHED) # write operation.xml to disk files_helper.write_operation_metadata(op) op_folder = files_helper.get_operation_folder(op.project.name, op.id) operation_xml = files_helper.get_operation_meta_file_path(op.project.name, op.id) op_folder_name = os.path.basename(op_folder) # add operation.xml zip_file.write(operation_xml, op_folder_name + '/' + os.path.basename(operation_xml)) # add linked datatypes to archive in the import operation for pth in linked_paths: zip_pth = op_folder_name + '/' + os.path.basename(pth) zip_file.write(pth, zip_pth) # remove these files, since we only want them in export archive files_helper.remove_folder(op_folder)
def _export_linked_datatypes(self, project, zip_file): files_helper = FilesHelper() linked_paths = self._get_linked_datatypes_storage_path(project) if not linked_paths: # do not export an empty operation return # Make a import operation which will contain links to other projects algo = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE, TVB_IMPORTER_CLASS) op = model.Operation(None, project.id, algo.id, '') op.project = project op.algorithm = algo op.id = 'links-to-external-projects' op.start_now() op.mark_complete(model.STATUS_FINISHED) # write operation.xml to disk files_helper.write_operation_metadata(op) op_folder = files_helper.get_operation_folder(op.project.name, op.id) operation_xml = files_helper.get_operation_meta_file_path(op.project.name, op.id) op_folder_name = os.path.basename(op_folder) # add operation.xml zip_file.write(operation_xml, op_folder_name + '/' + os.path.basename(operation_xml)) # add linked datatypes to archive in the import operation for pth in linked_paths: zip_pth = op_folder_name + '/' + os.path.basename(pth) zip_file.write(pth, zip_pth) # remove these files, since we only want them in export archive files_helper.remove_folder(op_folder)
def introduce_unmapped_node(out_pth, conn_zip_pth): """ Creates a connectivity with one extra node in the first position. This node represents the unmapped regions. :param out_pth: destination path :param conn_zip_pth: connectivity zip path. """ fh = FilesHelper() tmp_pth = os.path.splitext(out_pth)[0] fh.check_created(tmp_pth) files = fh.unpack_zip(conn_zip_pth, tmp_pth) for file_name in files: file_name_low = file_name.lower() if "centres" in file_name_low: with open(file_name) as f: lines = f.readlines() with open(file_name, 'w') as f: f.write("None 0.000000 0.000000 0.000000\n") f.writelines(lines) elif "weight" in file_name_low or "tract" in file_name_low: with open(file_name) as f: lines = f.readlines() nr_regions = len(lines) with open(file_name, 'w') as f: f.write(" 0.0000000e+00" * (nr_regions + 1) + '\n') for line in lines: f.write(" 0.0000000e+00" + line) else: raise Exception("this transformation does not support the file " + file_name) fh.zip_folder(out_pth, tmp_pth) fh.remove_folder(tmp_pth)
def introduce_unmapped_node(out_pth, conn_zip_pth): """ Creates a connectivity with one extra node in the first position. This node represents the unmapped regions. :param out_pth: destination path :param conn_zip_pth: connectivity zip path. """ fh = FilesHelper() tmp_pth = os.path.splitext(out_pth)[0] fh.check_created(tmp_pth) files = fh.unpack_zip(conn_zip_pth, tmp_pth) for file_name in files: file_name_low = file_name.lower() if "centres" in file_name_low: with open(file_name) as f: lines = f.readlines() with open(file_name, "w") as f: f.write("None 0.000000 0.000000 0.000000\n") f.writelines(lines) elif "weight" in file_name_low or "tract" in file_name_low: with open(file_name) as f: lines = f.readlines() nr_regions = len(lines) with open(file_name, "w") as f: f.write(" 0.0000000e+00" * (nr_regions + 1) + "\n") for line in lines: f.write(" 0.0000000e+00" + line) else: raise Exception("this transformation does not support the file " + file_name) fh.zip_folder(out_pth, tmp_pth) fh.remove_folder(tmp_pth)
class ProjectService: """ Services layer for Project entities. """ def __init__(self): self.logger = get_logger(__name__) self.structure_helper = FilesHelper() def store_project(self, current_user, is_create, selected_id, **data): """ We want to create/update a project entity. """ # Validate Unique Name new_name = data["name"] if len(new_name) < 1: raise ProjectServiceException("Invalid project name!") projects_no = dao.count_projects_for_name(new_name, selected_id) if projects_no > 0: err = {'name': 'Please choose another name, this one is used!'} raise formencode.Invalid("Duplicate Name Error", {}, None, error_dict=err) started_operations = dao.get_operation_numbers(selected_id)[1] if started_operations > 0: raise ProjectServiceException("A project can not be renamed while operations are still running!") if is_create: current_proj = Project(new_name, current_user.id, data["description"]) self.structure_helper.get_project_folder(current_proj) else: try: current_proj = dao.get_project_by_id(selected_id) except Exception as excep: self.logger.exception("An error has occurred!") raise ProjectServiceException(str(excep)) if current_proj.name != new_name: project_folder = self.structure_helper.get_project_folder(current_proj) if encryption_handler.encryption_enabled() and not encryption_handler.is_in_usage(project_folder): raise ProjectServiceException( "A project can not be renamed while sync encryption operations are running") self.structure_helper.rename_project_structure(current_proj.name, new_name) encrypted_path = encryption_handler.compute_encrypted_folder_path(project_folder) if os.path.exists(encrypted_path): new_encrypted_path = encryption_handler.compute_encrypted_folder_path( self.structure_helper.get_project_folder(new_name)) os.rename(encrypted_path, new_encrypted_path) current_proj.name = new_name current_proj.description = data["description"] # Commit to make sure we have a valid ID current_proj.refresh_update_date() self.structure_helper.write_project_metadata(current_proj) current_proj = dao.store_entity(current_proj) # Retrieve, to initialize lazy attributes current_proj = dao.get_project_by_id(current_proj.id) # Update share settings on current Project entity visited_pages = [] prj_admin = current_proj.administrator.username if 'visited_pages' in data and data['visited_pages']: visited_pages = data['visited_pages'].split(',') for page in visited_pages: members = UserService.retrieve_users_except([prj_admin], int(page), MEMBERS_PAGE_SIZE)[0] members = [m.id for m in members] dao.delete_members_for_project(current_proj.id, members) selected_user_ids = data["users"] if is_create and current_user.id not in selected_user_ids: # Make the project admin also member of the current project selected_user_ids.append(current_user.id) dao.add_members_to_project(current_proj.id, selected_user_ids) # Finish operation self.logger.debug("Edit/Save OK for project:" + str(current_proj.id) + ' by user:'******'-' result["count"] = one_op[2] result["gid"] = one_op[13] if one_op[3] is not None and one_op[3]: try: operation_group = dao.get_generic_entity(OperationGroup, one_op[3])[0] result["group"] = operation_group.name result["group"] = result["group"].replace("_", " ") result["operation_group_id"] = operation_group.id datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3]) result["datatype_group_gid"] = datatype_group.gid if datatype_group is not None else None result["gid"] = operation_group.gid # Filter only viewers for current DataTypeGroup entity: result["view_groups"] = AlgorithmService().get_visualizers_for_group(datatype_group.gid) \ if datatype_group is not None else None except Exception: self.logger.exception("We will ignore group on entity:" + str(one_op)) result["datatype_group_gid"] = None else: result['group'] = None result['datatype_group_gid'] = None result["algorithm"] = dao.get_algorithm_by_id(one_op[4]) result["user"] = dao.get_user_by_id(one_op[5]) if type(one_op[6]) is str: result["create"] = string2date(str(one_op[6])) else: result["create"] = one_op[6] if type(one_op[7]) is str: result["start"] = string2date(str(one_op[7])) else: result["start"] = one_op[7] if type(one_op[8]) is str: result["complete"] = string2date(str(one_op[8])) else: result["complete"] = one_op[8] if result["complete"] is not None and result["start"] is not None: result["duration"] = format_timedelta(result["complete"] - result["start"]) result["status"] = one_op[9] result["additional"] = one_op[10] result["visible"] = True if one_op[11] > 0 else False result['operation_tag'] = one_op[12] if not result['group']: datatype_results = dao.get_results_for_operation(result['id']) result['results'] = [] for dt in datatype_results: dt_loaded = load_entity_by_gid(dt.gid) if dt_loaded: result['results'].append(dt_loaded) else: self.logger.warning("Could not retrieve datatype %s" % str(dt)) else: result['results'] = None operations.append(result) except Exception: # We got an exception when processing one Operation Row. We will continue with the rest of the rows. self.logger.exception("Could not prepare operation for display:" + str(one_op)) return selected_project, total_ops_nr, operations, pages_no def retrieve_projects_for_user(self, user_id, current_page=1): """ Return a list with all Projects visible for current user. """ start_idx = PROJECTS_PAGE_SIZE * (current_page - 1) total = dao.get_projects_for_user(user_id, is_count=True) available_projects = dao.get_projects_for_user(user_id, start_idx, PROJECTS_PAGE_SIZE) pages_no = total // PROJECTS_PAGE_SIZE + (1 if total % PROJECTS_PAGE_SIZE else 0) for prj in available_projects: fns, sta, err, canceled, pending = dao.get_operation_numbers(prj.id) prj.operations_finished = fns prj.operations_started = sta prj.operations_error = err prj.operations_canceled = canceled prj.operations_pending = pending prj.disk_size = dao.get_project_disk_size(prj.id) prj.disk_size_human = format_bytes_human(prj.disk_size) self.logger.debug("Displaying " + str(len(available_projects)) + " projects in UI for user " + str(user_id)) return available_projects, pages_no @staticmethod def retrieve_all_user_projects(user_id, page_start=0, page_size=PROJECTS_PAGE_SIZE): """ Return a list with all projects visible for current user, without pagination. """ return dao.get_projects_for_user(user_id, page_start=page_start, page_size=page_size) @staticmethod def get_linkable_projects_for_user(user_id, data_id): """ Find projects with are visible for current user, and in which current datatype hasn't been linked yet. """ return dao.get_linkable_projects_for_user(user_id, data_id) @transactional def remove_project(self, project_id): """ Remove Project from DB and File Storage. """ try: project2delete = dao.get_project_by_id(project_id) self.logger.debug("Deleting project: id=" + str(project_id) + ' name=' + project2delete.name) project_datatypes = dao.get_datatypes_in_project(project_id) project_datatypes.sort(key=lambda dt: dt.create_date, reverse=True) for one_data in project_datatypes: self.remove_datatype(project_id, one_data.gid, True) links = dao.get_links_for_project(project_id) for one_link in links: dao.remove_entity(Links, one_link.id) project_bursts = dao.get_bursts_for_project(project_id) for burst in project_bursts: dao.remove_entity(burst.__class__, burst.id) project_folder = self.structure_helper.get_project_folder(project2delete) self.structure_helper.remove_project_structure(project2delete.name) encrypted_path = encryption_handler.compute_encrypted_folder_path(project_folder) if os.path.exists(encrypted_path): self.structure_helper.remove_folder(encrypted_path) if os.path.exists(encryption_handler.project_key_path(project_id)): os.remove(encryption_handler.project_key_path(project_id)) dao.delete_project(project_id) self.logger.debug("Deleted project: id=" + str(project_id) + ' name=' + project2delete.name) except RemoveDataTypeException as excep: self.logger.exception("Could not execute operation Node Remove!") raise ProjectServiceException(str(excep)) except FileStructureException as excep: self.logger.exception("Could not delete because of rights!") raise ProjectServiceException(str(excep)) except Exception as excep: self.logger.exception(str(excep)) raise ProjectServiceException(str(excep)) # ----------------- Methods for populating Data-Structure Page --------------- @staticmethod def get_datatype_in_group(group): """ Return all dataTypes that are the result of the same DTgroup. """ return dao.get_datatype_in_group(datatype_group_id=group) @staticmethod def get_datatypes_from_datatype_group(datatype_group_id): """ Retrieve all dataType which are part from the given dataType group. """ return dao.get_datatypes_from_datatype_group(datatype_group_id) @staticmethod def load_operation_by_gid(operation_gid): """ Retrieve loaded Operation from DB""" return dao.get_operation_by_gid(operation_gid) @staticmethod def load_operation_lazy_by_gid(operation_gid): """ Retrieve lazy Operation from DB""" return dao.get_operation_lazy_by_gid(operation_gid) @staticmethod def get_operation_group_by_id(operation_group_id): """ Loads OperationGroup from DB""" return dao.get_operationgroup_by_id(operation_group_id) @staticmethod def get_operation_group_by_gid(operation_group_gid): """ Loads OperationGroup from DB""" return dao.get_operationgroup_by_gid(operation_group_gid) @staticmethod def get_operations_in_group(operation_group): """ Return all the operations from an operation group. """ return dao.get_operations_in_group(operation_group.id) @staticmethod def is_upload_operation(operation_gid): """ Returns True only if the operation with the given GID is an upload operation. """ return dao.is_upload_operation(operation_gid) @staticmethod def get_all_operations_for_uploaders(project_id): """ Returns all finished upload operations. """ return dao.get_all_operations_for_uploaders(project_id) def set_operation_and_group_visibility(self, entity_gid, is_visible, is_operation_group=False): """ Sets the operation visibility. If 'is_operation_group' is True than this method will change the visibility for all the operation from the OperationGroup with the GID field equal to 'entity_gid'. """ def set_visibility(op): # workaround: # 'reload' the operation so that it has the project property set. # get_operations_in_group does not eager load it and now we're out of a sqlalchemy session # write_operation_metadata requires that property op = dao.get_operation_by_id(op.id) # end hack op.visible = is_visible dao.store_entity(op) def set_group_descendants_visibility(operation_group_id): ops_in_group = dao.get_operations_in_group(operation_group_id) for group_op in ops_in_group: set_visibility(group_op) if is_operation_group: op_group_id = dao.get_operationgroup_by_gid(entity_gid).id set_group_descendants_visibility(op_group_id) else: operation = dao.get_operation_by_gid(entity_gid) # we assure that if the operation belongs to a group than the visibility will be changed for the entire group if operation.fk_operation_group is not None: set_group_descendants_visibility(operation.fk_operation_group) else: set_visibility(operation) def get_operation_details(self, operation_gid, is_group): """ :returns: an entity OperationOverlayDetails filled with all information for current operation details. """ if is_group: operation_group = self.get_operation_group_by_gid(operation_gid) operation = dao.get_operations_in_group(operation_group.id, False, True) # Reload, to make sure all attributes lazy are populated as well. operation = dao.get_operation_by_gid(operation.gid) no_of_op_in_group = dao.get_operations_in_group(operation_group.id, is_count=True) datatype_group = self.get_datatypegroup_by_op_group_id(operation_group.id) count_result = dao.count_datatypes_in_group(datatype_group.id) else: operation = dao.get_operation_by_gid(operation_gid) if operation is None: return None no_of_op_in_group = 1 count_result = dao.count_resulted_datatypes(operation.id) user_display_name = dao.get_user_by_id(operation.fk_launched_by).display_name burst = dao.get_burst_for_operation_id(operation.id) datatypes_param, all_special_params = self._review_operation_inputs(operation.gid) op_pid = dao.get_operation_process_for_operation(operation.id) op_details = OperationOverlayDetails(operation, user_display_name, len(datatypes_param), count_result, burst, no_of_op_in_group, op_pid) # Add all parameter which are set differently by the user on this Operation. if all_special_params is not None: op_details.add_scientific_fields(all_special_params) return op_details @staticmethod def get_filterable_meta(): """ Contains all the attributes by which the user can structure the tree of DataTypes """ return DataTypeMetaData.get_filterable_meta() def get_project_structure(self, project, visibility_filter, first_level, second_level, filter_value): """ Find all DataTypes (including the linked ones and the groups) relevant for the current project. In case of a problem, will return an empty list. """ metadata_list = [] dt_list = dao.get_data_in_project(project.id, visibility_filter, filter_value) for dt in dt_list: # Prepare the DT results from DB, for usage in controller, by converting into DataTypeMetaData objects data = {} is_group = False group_op = None dt_entity = dao.get_datatype_by_gid(dt.gid) if dt_entity is None: self.logger.warning("Ignored entity (possibly removed DT class)" + str(dt)) continue # Filter by dt.type, otherwise Links to individual DT inside a group will be mistaken if dt.type == "DataTypeGroup" and dt.parent_operation.operation_group is not None: is_group = True group_op = dt.parent_operation.operation_group # All these fields are necessary here for dynamic Tree levels. data[DataTypeMetaData.KEY_DATATYPE_ID] = dt.id data[DataTypeMetaData.KEY_GID] = dt.gid data[DataTypeMetaData.KEY_NODE_TYPE] = dt.display_type data[DataTypeMetaData.KEY_STATE] = dt.state data[DataTypeMetaData.KEY_SUBJECT] = str(dt.subject) data[DataTypeMetaData.KEY_TITLE] = dt_entity.display_name data[DataTypeMetaData.KEY_RELEVANCY] = dt.visible data[DataTypeMetaData.KEY_LINK] = dt.parent_operation.fk_launched_in != project.id data[DataTypeMetaData.KEY_TAG_1] = dt.user_tag_1 if dt.user_tag_1 else '' data[DataTypeMetaData.KEY_TAG_2] = dt.user_tag_2 if dt.user_tag_2 else '' data[DataTypeMetaData.KEY_TAG_3] = dt.user_tag_3 if dt.user_tag_3 else '' data[DataTypeMetaData.KEY_TAG_4] = dt.user_tag_4 if dt.user_tag_4 else '' data[DataTypeMetaData.KEY_TAG_5] = dt.user_tag_5 if dt.user_tag_5 else '' # Operation related fields: operation_name = CommonDetails.compute_operation_name( dt.parent_operation.algorithm.algorithm_category.displayname, dt.parent_operation.algorithm.displayname) data[DataTypeMetaData.KEY_OPERATION_TYPE] = operation_name data[DataTypeMetaData.KEY_OPERATION_ALGORITHM] = dt.parent_operation.algorithm.displayname data[DataTypeMetaData.KEY_AUTHOR] = dt.parent_operation.user.username data[DataTypeMetaData.KEY_OPERATION_TAG] = group_op.name if is_group else dt.parent_operation.user_group data[DataTypeMetaData.KEY_OP_GROUP_ID] = group_op.id if is_group else None completion_date = dt.parent_operation.completion_date string_year = completion_date.strftime(MONTH_YEAR_FORMAT) if completion_date is not None else "" string_month = completion_date.strftime(DAY_MONTH_YEAR_FORMAT) if completion_date is not None else "" data[DataTypeMetaData.KEY_DATE] = date2string(completion_date) if (completion_date is not None) else '' data[DataTypeMetaData.KEY_CREATE_DATA_MONTH] = string_year data[DataTypeMetaData.KEY_CREATE_DATA_DAY] = string_month data[DataTypeMetaData.KEY_BURST] = dt._parent_burst.name if dt._parent_burst is not None else '-None-' metadata_list.append(DataTypeMetaData(data, dt.invalid)) return StructureNode.metadata2tree(metadata_list, first_level, second_level, project.id, project.name) @staticmethod def get_datatype_details(datatype_gid): """ :returns: an array. First entry in array is an instance of DataTypeOverlayDetails\ The second one contains all the possible states for the specified dataType. """ meta_atts = DataTypeOverlayDetails() states = DataTypeMetaData.STATES try: datatype_result = dao.get_datatype_details(datatype_gid) meta_atts.fill_from_datatype(datatype_result, datatype_result._parent_burst) return meta_atts, states, datatype_result except Exception: # We ignore exception here (it was logged above, and we want to return no details). return meta_atts, states, None def _remove_project_node_files(self, project_id, gid, skip_validation=False): """ Delegate removal of a node in the structure of the project. In case of a problem will THROW StructureException. """ try: project = self.find_project(project_id) datatype = dao.get_datatype_by_gid(gid) links = dao.get_links_for_datatype(datatype.id) op = dao.get_operation_by_id(datatype.fk_from_operation) adapter = ABCAdapter.build_adapter(op.algorithm) if links: was_link = False for link in links: # This means it's only a link and we need to remove it if link.fk_from_datatype == datatype.id and link.fk_to_project == project.id: dao.remove_entity(Links, link.id) was_link = True if not was_link: # Create a clone of the operation # There is no view_model so the view_model_gid is None new_op = Operation(op.view_model_gid, dao.get_system_user().id, links[0].fk_to_project, datatype.parent_operation.fk_from_algo, datatype.parent_operation.status, datatype.parent_operation.start_date, datatype.parent_operation.completion_date, datatype.parent_operation.fk_operation_group, datatype.parent_operation.additional_info, datatype.parent_operation.user_group, datatype.parent_operation.range_values) new_op = dao.store_entity(new_op) to_project = self.find_project(links[0].fk_to_project) to_project_path = self.structure_helper.get_project_folder(to_project) encryption_handler.set_project_active(to_project) encryption_handler.sync_folders(to_project_path) to_project_name = to_project.name full_path = h5.path_for_stored_index(datatype) self.structure_helper.move_datatype(datatype, to_project_name, str(new_op.id), full_path) # Move also the ViewModel H5 old_folder = self.structure_helper.get_project_folder(project, str(op.id)) view_model = adapter.load_view_model(op) vm_full_path = h5.determine_filepath(op.view_model_gid, old_folder) self.structure_helper.move_datatype(view_model, to_project_name, str(new_op.id), vm_full_path) encryption_handler.sync_folders(to_project_path) encryption_handler.set_project_inactive(to_project) datatype.fk_from_operation = new_op.id datatype.parent_operation = new_op dao.store_entity(datatype) dao.remove_entity(Links, links[0].id) else: specific_remover = get_remover(datatype.type)(datatype) specific_remover.remove_datatype(skip_validation) h5_path = h5.path_for_stored_index(datatype) self.structure_helper.remove_datatype_file(h5_path) encryption_handler.push_folder_to_sync(self.structure_helper.get_project_folder_from_h5(h5_path)) except RemoveDataTypeException: self.logger.exception("Could not execute operation Node Remove!") raise except FileStructureException: self.logger.exception("Remove operation failed") raise StructureException("Remove operation failed for unknown reasons.Please contact system administrator.") def remove_operation(self, operation_id): """ Remove a given operation """ operation = dao.try_get_operation_by_id(operation_id) if operation is not None: self.logger.debug("Deleting operation %s " % operation) datatypes_for_op = dao.get_results_for_operation(operation_id) for dt in reversed(datatypes_for_op): self.remove_datatype(operation.project.id, dt.gid, False) # Here the Operation is mot probably already removed - in case DTs were found inside # but we still remove it for the case when no DTs exist dao.remove_entity(Operation, operation.id) self.structure_helper.remove_operation_data(operation.project.name, operation_id) encryption_handler.push_folder_to_sync(self.structure_helper.get_project_folder(operation.project)) self.logger.debug("Finished deleting operation %s " % operation) else: self.logger.warning("Attempt to delete operation with id=%s which no longer exists." % operation_id) def remove_datatype(self, project_id, datatype_gid, skip_validation=False): """ Method used for removing a dataType. If the given dataType is a DatatypeGroup or a dataType from a DataTypeGroup than this method will remove the entire group. The operation(s) used for creating the dataType(s) will also be removed. """ datatype = dao.get_datatype_by_gid(datatype_gid) if datatype is None: self.logger.warning("Attempt to delete DT[%s] which no longer exists." % datatype_gid) return is_datatype_group = False datatype_group = None if dao.is_datatype_group(datatype_gid): is_datatype_group = True datatype_group = datatype elif datatype.fk_datatype_group is not None: is_datatype_group = True datatype_group = dao.get_datatype_by_id(datatype.fk_datatype_group) operations_set = [datatype.fk_from_operation] correct = True if is_datatype_group: operations_set = [datatype_group.fk_from_operation] self.logger.debug("Removing datatype group %s" % datatype_group) if datatype_group.fk_parent_burst: burst = dao.get_generic_entity(BurstConfiguration, datatype_group.fk_parent_burst, 'gid')[0] dao.remove_entity(BurstConfiguration, burst.id) if burst.fk_metric_operation_group: correct = correct and self._remove_operation_group(burst.fk_metric_operation_group, project_id, skip_validation, operations_set) if burst.fk_operation_group: correct = correct and self._remove_operation_group(burst.fk_operation_group, project_id, skip_validation, operations_set) else: self._remove_datatype_group_dts(project_id, datatype_group.id, skip_validation, operations_set) datatype_group = dao.get_datatype_group_by_gid(datatype_group.gid) dao.remove_entity(DataTypeGroup, datatype.id) correct = correct and dao.remove_entity(OperationGroup, datatype_group.fk_operation_group) else: self.logger.debug("Removing datatype %s" % datatype) self._remove_project_node_files(project_id, datatype.gid, skip_validation) # Remove Operation entity in case no other DataType needs them. project = dao.get_project_by_id(project_id) for operation_id in operations_set: dependent_dt = dao.get_generic_entity(DataType, operation_id, "fk_from_operation") if len(dependent_dt) > 0: # Do not remove Operation in case DataType still exist referring it. continue op_burst = dao.get_burst_for_operation_id(operation_id) if op_burst: correct = correct and dao.remove_entity(BurstConfiguration, op_burst.id) correct = correct and dao.remove_entity(Operation, operation_id) # Make sure Operation folder is removed self.structure_helper.remove_operation_data(project.name, operation_id) encryption_handler.push_folder_to_sync(self.structure_helper.get_project_folder(project)) if not correct: raise RemoveDataTypeException("Could not remove DataType " + str(datatype_gid)) def _remove_operation_group(self, operation_group_id, project_id, skip_validation, operations_set): metrics_groups = dao.get_generic_entity(DataTypeGroup, operation_group_id, 'fk_operation_group') if len(metrics_groups) > 0: metric_datatype_group_id = metrics_groups[0].id self._remove_datatype_group_dts(project_id, metric_datatype_group_id, skip_validation, operations_set) dao.remove_entity(DataTypeGroup, metric_datatype_group_id) return dao.remove_entity(OperationGroup, operation_group_id) def _remove_datatype_group_dts(self, project_id, dt_group_id, skip_validation, operations_set): data_list = dao.get_datatypes_from_datatype_group(dt_group_id) for adata in data_list: self._remove_project_node_files(project_id, adata.gid, skip_validation) if adata.fk_from_operation not in operations_set: operations_set.append(adata.fk_from_operation) def update_metadata(self, submit_data): """ Update DataType/ DataTypeGroup metadata THROW StructureException when input data is invalid. """ new_data = dict() for key in DataTypeOverlayDetails().meta_attributes_list: if key in submit_data: value = submit_data[key] if value == "None": value = None if value == "" and key in [CommonDetails.CODE_OPERATION_TAG, CommonDetails.CODE_OPERATION_GROUP_ID]: value = None new_data[key] = value try: if (CommonDetails.CODE_OPERATION_GROUP_ID in new_data and new_data[CommonDetails.CODE_OPERATION_GROUP_ID]): # We need to edit a group all_data_in_group = dao.get_datatype_in_group(operation_group_id= new_data[CommonDetails.CODE_OPERATION_GROUP_ID]) if len(all_data_in_group) < 1: raise StructureException("Inconsistent group, can not be updated!") # datatype_group = dao.get_generic_entity(DataTypeGroup, all_data_in_group[0].fk_datatype_group)[0] # all_data_in_group.append(datatype_group) for datatype in all_data_in_group: self._edit_data(datatype, new_data, True) else: # Get the required DataType and operation from DB to store changes that will be done in XML. gid = new_data[CommonDetails.CODE_GID] datatype = dao.get_datatype_by_gid(gid) self._edit_data(datatype, new_data) except Exception as excep: self.logger.exception(excep) raise StructureException(str(excep)) def _edit_data(self, datatype, new_data, from_group=False): # type: (DataType, dict, bool) -> None """ Private method, used for editing a meta-data XML file and a DataType row for a given custom DataType entity with new dictionary of data from UI. """ # 1. First update Operation fields: # Update group field if possible new_group_name = new_data[CommonDetails.CODE_OPERATION_TAG] empty_group_value = (new_group_name is None or new_group_name == "") if from_group: if empty_group_value: raise StructureException("Empty group is not allowed!") group = dao.get_generic_entity(OperationGroup, new_data[CommonDetails.CODE_OPERATION_GROUP_ID]) if group and len(group) > 0 and new_group_name != group[0].name: group = group[0] exists_group = dao.get_generic_entity(OperationGroup, new_group_name, 'name') if exists_group: raise StructureException("Group '" + new_group_name + "' already exists.") group.name = new_group_name dao.store_entity(group) else: operation = dao.get_operation_by_id(datatype.fk_from_operation) operation.user_group = new_group_name dao.store_entity(operation) op_folder = self.structure_helper.get_project_folder(operation.project, str(operation.id)) vm_gid = operation.view_model_gid view_model_file = h5.determine_filepath(vm_gid, op_folder) if view_model_file: view_model_class = H5File.determine_type(view_model_file) view_model = view_model_class() with ViewModelH5(view_model_file, view_model) as f: ga = f.load_generic_attributes() ga.operation_tag = new_group_name f.store_generic_attributes(ga, False) else: self.logger.warning("Could not find ViewModel H5 file for op: {}".format(operation)) # 2. Update GenericAttributes in the associated H5 files: h5_path = h5.path_for_stored_index(datatype) with H5File.from_file(h5_path) as f: ga = f.load_generic_attributes() ga.subject = new_data[DataTypeOverlayDetails.DATA_SUBJECT] ga.state = new_data[DataTypeOverlayDetails.DATA_STATE] ga.operation_tag = new_group_name if DataTypeOverlayDetails.DATA_TAG_1 in new_data: ga.user_tag_1 = new_data[DataTypeOverlayDetails.DATA_TAG_1] if DataTypeOverlayDetails.DATA_TAG_2 in new_data: ga.user_tag_2 = new_data[DataTypeOverlayDetails.DATA_TAG_2] if DataTypeOverlayDetails.DATA_TAG_3 in new_data: ga.user_tag_3 = new_data[DataTypeOverlayDetails.DATA_TAG_3] if DataTypeOverlayDetails.DATA_TAG_4 in new_data: ga.user_tag_4 = new_data[DataTypeOverlayDetails.DATA_TAG_4] if DataTypeOverlayDetails.DATA_TAG_5 in new_data: ga.user_tag_5 = new_data[DataTypeOverlayDetails.DATA_TAG_5] f.store_generic_attributes(ga, False) # 3. Update MetaData in DT Index DB as well. datatype.fill_from_generic_attributes(ga) dao.store_entity(datatype) def get_datatype_and_datatypegroup_inputs_for_operation(self, operation_gid, selected_filter): """ Returns the dataTypes that are used as input parameters for the given operation. 'selected_filter' - is expected to be a visibility filter. If any dataType is part of a dataType group then the dataType group will be returned instead of that dataType. """ all_datatypes = self._review_operation_inputs(operation_gid)[0] datatype_inputs = [] for datatype in all_datatypes: if selected_filter.display_name == StaticFiltersFactory.RELEVANT_VIEW: if datatype.visible: datatype_inputs.append(datatype) else: datatype_inputs.append(datatype) datatypes = [] datatype_groups = dict() for data_type in datatype_inputs: if data_type.fk_datatype_group is None: datatypes.append(data_type) elif data_type.fk_datatype_group not in datatype_groups: dt_group = dao.get_datatype_by_id(data_type.fk_datatype_group) datatype_groups[data_type.fk_datatype_group] = dt_group datatypes.extend([v for v in datatype_groups.values()]) return datatypes def _review_operation_inputs(self, operation_gid): """ :returns: A list of DataTypes that are used as input parameters for the specified operation. And a dictionary will all operation parameters different then the default ones. """ operation = dao.get_operation_by_gid(operation_gid) try: adapter = ABCAdapter.build_adapter(operation.algorithm) return review_operation_inputs_from_adapter(adapter, operation) except Exception: self.logger.exception("Could not load details for operation %s" % operation_gid) if operation.view_model_gid: changed_parameters = dict(Warning="Algorithm changed dramatically. We can not offer more details") else: changed_parameters = dict(Warning="GID parameter is missing. Old implementation of the operation.") return [], changed_parameters @staticmethod def get_results_for_operation(operation_id, selected_filter=None): """ Retrieve the DataTypes entities resulted after the execution of the given operation. """ return dao.get_results_for_operation(operation_id, selected_filter) @staticmethod def get_datatype_by_id(datatype_id): """Retrieve a DataType DB reference by its id.""" return dao.get_datatype_by_id(datatype_id) @staticmethod def get_datatypegroup_by_gid(datatypegroup_gid): """ Returns the DataTypeGroup with the specified gid. """ return dao.get_datatype_group_by_gid(datatypegroup_gid) @staticmethod def get_datatypegroup_by_op_group_id(operation_group_id): """ Returns the DataTypeGroup with the specified id. """ return dao.get_datatypegroup_by_op_group_id(operation_group_id) @staticmethod def get_datatypes_in_project(project_id, only_visible=False): return dao.get_data_in_project(project_id, only_visible) @staticmethod def set_datatype_visibility(datatype_gid, is_visible): """ Sets the dataType visibility. If the given dataType is a dataType group or it is part of a dataType group than this method will set the visibility for each dataType from this group. """ def set_visibility(dt): """ set visibility flag, persist in db and h5""" dt.visible = is_visible dt = dao.store_entity(dt) h5_path = h5.path_for_stored_index(dt) with H5File.from_file(h5_path) as f: f.visible.store(is_visible) def set_group_descendants_visibility(datatype_group_id): datatypes_in_group = dao.get_datatypes_from_datatype_group(datatype_group_id) for group_dt in datatypes_in_group: set_visibility(group_dt) datatype = dao.get_datatype_by_gid(datatype_gid) if isinstance(datatype, DataTypeGroup): # datatype is a group set_group_descendants_visibility(datatype.id) datatype.visible = is_visible dao.store_entity(datatype) elif datatype.fk_datatype_group is not None: # datatype is member of a group set_group_descendants_visibility(datatype.fk_datatype_group) # the datatype to be updated is the parent datatype group parent = dao.get_datatype_by_id(datatype.fk_datatype_group) parent.visible = is_visible dao.store_entity(parent) else: # update the single datatype. set_visibility(datatype) @staticmethod def is_datatype_group(datatype_gid): """ Used to check if the dataType with the specified GID is a DataTypeGroup. """ return dao.is_datatype_group(datatype_gid) def get_linked_datatypes_storage_path(self, project): """ :return: the file paths to the datatypes that are linked in `project` """ paths = [] for lnk_dt in dao.get_linked_datatypes_in_project(project.id): # get datatype as a mapped type lnk_dt = dao.get_datatype_by_gid(lnk_dt.gid) path = h5.path_for_stored_index(lnk_dt) if path is not None: paths.append(path) else: self.logger.warning("Problem when trying to retrieve path on %s:%s!" % (lnk_dt.type, lnk_dt.gid)) return paths
class FilesHelperTest(TransactionalTestCase): """ This class contains tests for the tvb.core.entities.file.files_helper module. """ PROJECT_NAME = "test_proj" def setUp(self): """ Set up the context needed by the tests. """ self.files_helper = FilesHelper() self.test_user = TestFactory.create_user() self.test_project = TestFactory.create_project(self.test_user, self.PROJECT_NAME) def tearDown(self): """ Remove generated project during tests. """ self.delete_project_folders() def test_check_created(self): """ Test standard flows for check created. """ self.files_helper.check_created() self.assertTrue(os.path.exists(root_storage), "Storage not created!") self.files_helper.check_created(os.path.join(root_storage, "test")) self.assertTrue(os.path.exists(root_storage), "Storage not created!") self.assertTrue(os.path.exists(os.path.join(root_storage, "test")), "Test directory not created!") def test_get_project_folder(self): """ Test the get_project_folder method which should create a folder in case it doesn't already exist. """ project_path = self.files_helper.get_project_folder(self.test_project) self.assertTrue(os.path.exists(project_path), "Folder doesn't exist") folder_path = self.files_helper.get_project_folder(self.test_project, "43") self.assertTrue(os.path.exists(project_path), "Folder doesn't exist") self.assertTrue(os.path.exists(folder_path), "Folder doesn't exist") def test_rename_project_structure(self): """ Try to rename the folder structure of a project. Standard flow. """ self.files_helper.get_project_folder(self.test_project) path, name = self.files_helper.rename_project_structure(self.test_project.name, "new_name") self.assertNotEqual(path, name, "Rename didn't take effect.") def test_rename_structure_same_name(self): """ Try to rename the folder structure of a project. Same name. """ self.files_helper.get_project_folder(self.test_project) self.assertRaises(FileStructureException, self.files_helper.rename_project_structure, self.test_project.name, self.PROJECT_NAME) def test_remove_project_structure(self): """ Check that remove project structure deletes the corresponding folder. Standard flow. """ full_path = self.files_helper.get_project_folder(self.test_project) self.assertTrue(os.path.exists(full_path), "Folder was not created.") self.files_helper.remove_project_structure(self.test_project.name) self.assertFalse(os.path.exists(full_path), "Project folder not deleted.") def test_write_project_metadata(self): """ Write XML for test-project. """ self.files_helper.write_project_metadata(self.test_project) expected_file = self.files_helper.get_project_meta_file_path(self.PROJECT_NAME) self.assertTrue(os.path.exists(expected_file)) project_meta = XMLReader(expected_file).read_metadata() loaded_project = model.Project(None, None) loaded_project.from_dict(project_meta, self.test_user.id) self.assertEqual(self.test_project.name, loaded_project.name) self.assertEqual(self.test_project.description, loaded_project.description) self.assertEqual(self.test_project.gid, loaded_project.gid) expected_dict = self.test_project.to_dict()[1] del expected_dict['last_updated'] found_dict = loaded_project.to_dict()[1] del found_dict['last_updated'] self.assertDictContainsSubset(expected_dict, found_dict) self.assertDictContainsSubset(found_dict, expected_dict) def test_write_operation_metadata(self): """ Test that a correct XML is created for an operation. """ operation = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project) expected_file = self.files_helper.get_operation_meta_file_path(self.PROJECT_NAME, operation.id) self.assertFalse(os.path.exists(expected_file)) self.files_helper.write_operation_metadata(operation) self.assertTrue(os.path.exists(expected_file)) operation_meta = XMLReader(expected_file).read_metadata() loaded_operation = model.Operation(None, None, None, None) loaded_operation.from_dict(operation_meta, dao) expected_dict = operation.to_dict()[1] found_dict = loaded_operation.to_dict()[1] for key, value in expected_dict.iteritems(): self.assertEqual(str(value), str(found_dict[key])) # Now validate that operation metaData can be also updated self.assertNotEqual("new_group_name", found_dict['user_group']) self.files_helper.update_operation_metadata(self.PROJECT_NAME, "new_group_name", operation.id) found_dict = XMLReader(expected_file).read_metadata() self.assertEqual("new_group_name", found_dict['user_group']) def test_remove_dt_happy_flow(self): """ Happy flow for removing a file related to a DataType. """ folder_path = self.files_helper.get_project_folder(self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path open(datatype.get_storage_file_path(), 'w') self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") self.files_helper.remove_datatype(datatype) self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not deleted!") def test_remove_dt_non_existent(self): """ Try to call remove on a dataType with no H5 file. Should work. """ folder_path = self.files_helper.get_project_folder(self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path self.assertFalse(os.path.exists(datatype.get_storage_file_path())) self.files_helper.remove_datatype(datatype) def test_move_datatype(self): """ Make sure associated H5 file is moved to a correct new location. """ folder_path = self.files_helper.get_project_folder(self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path open(datatype.get_storage_file_path(), 'w') self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") self.files_helper.move_datatype(datatype, self.PROJECT_NAME + '11', "43") self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not moved!") datatype.storage_path = self.files_helper.get_project_folder(self.PROJECT_NAME + '11', "43") self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") def test_find_relative_path(self): """ Tests that relative path is computed properly. """ rel_path = self.files_helper.find_relative_path("/root/up/to/here/test/it/now", "/root/up/to/here") self.assertEqual(rel_path, os.sep.join(["test", "it", "now"]), "Did not extract relative path as expected.") def test_remove_files_valid(self): """ Pass a valid list of files and check they are all removed. """ file_list = ["test1", "test2", "test3"] for file_n in file_list: fp = open(file_n, 'w') fp.write('test') fp.close() for file_n in file_list: self.assertTrue(os.path.isfile(file_n)) self.files_helper.remove_files(file_list) for file_n in file_list: self.assertFalse(os.path.isfile(file_n)) def test_remove_folder(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" os.mkdir(folder_name) self.assertTrue(os.path.isdir(folder_name), "Folder should be created.") self.files_helper.remove_folder(folder_name) self.assertFalse(os.path.isdir(folder_name), "Folder should be deleted.") def test_remove_folder_non_existing_ignore_exc(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.") self.files_helper.remove_folder(folder_name, ignore_errors=True) def test_remove_folder_non_existing(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.") self.assertRaises(FileStructureException, self.files_helper.remove_folder, folder_name, False)
class TestFilesHelper(TransactionalTestCase): """ This class contains tests for the tvb.core.entities.file.files_helper module. """ PROJECT_NAME = "test_proj" def transactional_setup_method(self): """ Set up the context needed by the tests. """ self.files_helper = FilesHelper() self.test_user = TestFactory.create_user() self.test_project = TestFactory.create_project(self.test_user, self.PROJECT_NAME) def transactional_teardown_method(self): """ Remove generated project during tests. """ self.delete_project_folders() def test_check_created(self): """ Test standard flows for check created. """ self.files_helper.check_created() assert os.path.exists(root_storage), "Storage not created!" self.files_helper.check_created(os.path.join(root_storage, "test")) assert os.path.exists(root_storage), "Storage not created!" assert os.path.exists(os.path.join(root_storage, "test")), "Test directory not created!" def test_get_project_folder(self): """ Test the get_project_folder method which should create a folder in case it doesn't already exist. """ project_path = self.files_helper.get_project_folder(self.test_project) assert os.path.exists(project_path), "Folder doesn't exist" folder_path = self.files_helper.get_project_folder(self.test_project, "43") assert os.path.exists(project_path), "Folder doesn't exist" assert os.path.exists(folder_path), "Folder doesn't exist" def test_rename_project_structure(self): """ Try to rename the folder structure of a project. Standard flow. """ self.files_helper.get_project_folder(self.test_project) path, name = self.files_helper.rename_project_structure(self.test_project.name, "new_name") assert path != name, "Rename didn't take effect." def test_rename_structure_same_name(self): """ Try to rename the folder structure of a project. Same name. """ self.files_helper.get_project_folder(self.test_project) with pytest.raises(FileStructureException): self.files_helper.rename_project_structure(self.test_project.name, self.PROJECT_NAME) def test_remove_project_structure(self): """ Check that remove project structure deletes the corresponding folder. Standard flow. """ full_path = self.files_helper.get_project_folder(self.test_project) assert os.path.exists(full_path), "Folder was not created." self.files_helper.remove_project_structure(self.test_project.name) assert not os.path.exists(full_path), "Project folder not deleted." def test_write_project_metadata(self): """ Write XML for test-project. """ self.files_helper.write_project_metadata(self.test_project) expected_file = self.files_helper.get_project_meta_file_path(self.PROJECT_NAME) assert os.path.exists(expected_file) project_meta = XMLReader(expected_file).read_metadata() loaded_project = model_project.Project(None, None) loaded_project.from_dict(project_meta, self.test_user.id) assert self.test_project.name == loaded_project.name assert self.test_project.description == loaded_project.description assert self.test_project.gid == loaded_project.gid expected_dict = self.test_project.to_dict()[1] del expected_dict['last_updated'] found_dict = loaded_project.to_dict()[1] del found_dict['last_updated'] self._dictContainsSubset(expected_dict, found_dict) self._dictContainsSubset(found_dict, expected_dict) def test_write_operation_metadata(self): """ Test that a correct XML is created for an operation. """ operation = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project) expected_file = self.files_helper.get_operation_meta_file_path(self.PROJECT_NAME, operation.id) assert not os.path.exists(expected_file) self.files_helper.write_operation_metadata(operation) assert os.path.exists(expected_file) operation_meta = XMLReader(expected_file).read_metadata() loaded_operation = model_operation.Operation(None, None, None, None) loaded_operation.from_dict(operation_meta, dao, user_id=self.test_user.id) expected_dict = operation.to_dict()[1] found_dict = loaded_operation.to_dict()[1] for key, value in expected_dict.items(): assert str(value) == str(found_dict[key]) # Now validate that operation metaData can be also updated assert "new_group_name" != found_dict['user_group'] self.files_helper.update_operation_metadata(self.PROJECT_NAME, "new_group_name", operation.id) found_dict = XMLReader(expected_file).read_metadata() assert "new_group_name" == found_dict['user_group'] def test_remove_dt_happy_flow(self, dummy_datatype_index_factory): """ Happy flow for removing a file related to a DataType. """ datatype = dummy_datatype_index_factory() h5_path = h5.path_for_stored_index(datatype) assert os.path.exists(h5_path), "Test file was not created!" self.files_helper.remove_datatype_file(h5_path) assert not os.path.exists(h5_path), "Test file was not deleted!" def test_remove_dt_non_existent(self, dummy_datatype_index_factory): """ Try to call remove on a dataType with no H5 file. Should work. """ datatype = dummy_datatype_index_factory() h5_path = h5.path_for_stored_index(datatype) wrong_path = os.path.join(h5_path, "WRONG_PATH") assert not os.path.exists(wrong_path) self.files_helper.remove_datatype_file(wrong_path) def test_move_datatype(self, dummy_datatype_index_factory): """ Make sure associated H5 file is moved to a correct new location. """ datatype = dummy_datatype_index_factory(project=self.test_project) old_file_path = h5.path_for_stored_index(datatype) assert os.path.exists(old_file_path), "Test file was not created!" full_path = h5.path_for_stored_index(datatype) self.files_helper.move_datatype(datatype, self.PROJECT_NAME + '2', "1", full_path) assert not os.path.exists(old_file_path), "Test file was not moved!" datatype.fk_from_operation = 43 new_file_path = os.path.join(self.files_helper.get_project_folder(self.PROJECT_NAME + '2', "1"), old_file_path.split("\\")[-1]) assert os.path.exists(new_file_path), "Test file was not created!" def test_find_relative_path(self): """ Tests that relative path is computed properly. """ rel_path = self.files_helper.find_relative_path("/root/up/to/here/test/it/now", "/root/up/to/here") assert rel_path == os.sep.join(["test", "it", "now"]), "Did not extract relative path as expected." def test_remove_files_valid(self): """ Pass a valid list of files and check they are all removed. """ file_list = ["test1", "test2", "test3"] for file_n in file_list: fp = open(file_n, 'w') fp.write('test') fp.close() for file_n in file_list: assert os.path.isfile(file_n) self.files_helper.remove_files(file_list) for file_n in file_list: assert not os.path.isfile(file_n) def test_remove_folder(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" os.mkdir(folder_name) assert os.path.isdir(folder_name), "Folder should be created." self.files_helper.remove_folder(folder_name) assert not os.path.isdir(folder_name), "Folder should be deleted." def test_remove_folder_non_existing_ignore_exc(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" assert not os.path.isdir(folder_name), "Folder should not exist before call." self.files_helper.remove_folder(folder_name, ignore_errors=True) def test_remove_folder_non_existing(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" assert not os.path.isdir(folder_name), "Folder should not exist before call." with pytest.raises(FileStructureException): self.files_helper.remove_folder(folder_name, False) def _dictContainsSubset(self, expected, actual, msg=None): """Checks whether actual is a superset of expected.""" missing = [] mismatched = [] for key, value in expected.items(): if key not in actual: return False elif value != actual[key]: return False return True
class ExportManager(object): """ This class provides basic methods for exporting data types of projects in different formats. """ all_exporters = {} # Dictionary containing all available exporters export_folder = None EXPORT_FOLDER_NAME = "EXPORT_TMP" EXPORTED_SIMULATION_NAME = "exported_simulation" EXPORTED_SIMULATION_DTS_DIR = "datatypes" ZIP_FILE_EXTENSION = "zip" logger = get_logger(__name__) def __init__(self): # Here we register all available data type exporters # If new exporters supported, they should be added here self._register_exporter(TVBExporter()) self._register_exporter(TVBLinkedExporter()) self.export_folder = os.path.join(TvbProfile.current.TVB_STORAGE, self.EXPORT_FOLDER_NAME) self.files_helper = FilesHelper() def _register_exporter(self, exporter): """ This method register into an internal format available exporters. :param exporter: Instance of a data type exporter (extends ABCExporter) """ if exporter is not None: self.all_exporters[exporter.__class__.__name__] = exporter def get_exporters_for_data(self, data): """ Get available exporters for current data type. :returns: a dictionary with the {exporter_id : label} """ if data is None: raise InvalidExportDataException( "Could not detect exporters for null data") self.logger.debug("Trying to determine exporters valid for %s" % data.type) results = {} # No exporter for None data if data is None: return results for exporterId in self.all_exporters.keys(): exporter = self.all_exporters[exporterId] if exporter.accepts(data): results[exporterId] = exporter.get_label() return results def export_data(self, data, exporter_id, project): """ Export provided data using given exporter :param data: data type to be exported :param exporter_id: identifier of the exporter to be used :param project: project that contains data to be exported :returns: a tuple with the following elements 1. name of the file to be shown to user 2. full path of the export file (available for download) 3. boolean which specify if file can be deleted after download """ if data is None: raise InvalidExportDataException( "Could not export null data. Please select data to be exported" ) if exporter_id is None: raise ExportException( "Please select the exporter to be used for this operation") if exporter_id not in self.all_exporters: raise ExportException( "Provided exporter identifier is not a valid one") exporter = self.all_exporters[exporter_id] if project is None: raise ExportException( "Please provide the project where data files are stored") # Now we start the real export if not exporter.accepts(data): raise InvalidExportDataException( "Current data can not be exported by specified exporter") # Now compute and create folder where to store exported data # This will imply to generate a folder which is unique for each export data_export_folder = None try: data_export_folder = self._build_data_export_folder(data) self.logger.debug("Start export of data: %s" % data.type) export_data = exporter.export(data, data_export_folder, project) finally: # In case export did not generated any file delete folder if data_export_folder is not None and len( os.listdir(data_export_folder)) == 0: os.rmdir(data_export_folder) return export_data def _export_linked_datatypes(self, project, zip_file): linked_paths = ProjectService().get_linked_datatypes_storage_path( project) if not linked_paths: # do not export an empty operation return # Make a import operation which will contain links to other projects algo = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE, TVB_IMPORTER_CLASS) op = model_operation.Operation(None, None, project.id, algo.id) op.project = project op.algorithm = algo op.id = 'links-to-external-projects' op.start_now() op.mark_complete(model_operation.STATUS_FINISHED) op_folder = self.files_helper.get_operation_folder( op.project.name, op.id) op_folder_name = os.path.basename(op_folder) # add linked datatypes to archive in the import operation for pth in linked_paths: zip_pth = op_folder_name + '/' + os.path.basename(pth) zip_file.write(pth, zip_pth) # remove these files, since we only want them in export archive self.files_helper.remove_folder(op_folder) def export_project(self, project, optimize_size=False): """ Given a project root and the TVB storage_path, create a ZIP ready for export. :param project: project object which identifies project to be exported """ if project is None: raise ExportException("Please provide project to be exported") project_folder = self.files_helper.get_project_folder(project) project_datatypes = dao.get_datatypes_in_project( project.id, only_visible=optimize_size) to_be_exported_folders = [] considered_op_ids = [] folders_to_exclude = self._get_op_with_errors(project.id) if optimize_size: # take only the DataType with visibility flag set ON for dt in project_datatypes: op_id = dt.fk_from_operation if op_id not in considered_op_ids: to_be_exported_folders.append({ 'folder': self.files_helper.get_project_folder( project, str(op_id)), 'archive_path_prefix': str(op_id) + os.sep, 'exclude': folders_to_exclude }) considered_op_ids.append(op_id) else: folders_to_exclude.append("TEMP") to_be_exported_folders.append({ 'folder': project_folder, 'archive_path_prefix': '', 'exclude': folders_to_exclude }) # Compute path and name of the zip file now = datetime.now() date_str = now.strftime("%Y-%m-%d_%H-%M") zip_file_name = "%s_%s.%s" % (date_str, project.name, self.ZIP_FILE_EXTENSION) export_folder = self._build_data_export_folder(project) result_path = os.path.join(export_folder, zip_file_name) with TvbZip(result_path, "w") as zip_file: # Pack project [filtered] content into a ZIP file: self.logger.debug("Done preparing, now we will write folders " + str(len(to_be_exported_folders))) self.logger.debug(str(to_be_exported_folders)) for pack in to_be_exported_folders: zip_file.write_folder(**pack) self.logger.debug( "Done exporting files, now we will export linked DTs") self._export_linked_datatypes(project, zip_file) # Make sure the Project.xml file gets copied: if optimize_size: self.logger.debug("Done linked, now we write the project xml") zip_file.write( self.files_helper.get_project_meta_file_path(project.name), self.files_helper.TVB_PROJECT_FILE) self.logger.debug("Done, closing") return result_path @staticmethod def _get_op_with_errors(project_id): """ Get the operation folders with error base name as list. """ operations = dao.get_operations_with_error_in_project(project_id) op_with_errors = [] for op in operations: op_with_errors.append(op.id) return op_with_errors def _build_data_export_folder(self, data): """ This method computes the folder where results of an export operation will be stored for a while (e.g until download is done; or for 1 day) """ now = datetime.now() date_str = "%d-%d-%d_%d-%d-%d_%d" % (now.year, now.month, now.day, now.hour, now.minute, now.second, now.microsecond) tmp_str = date_str + "@" + data.gid data_export_folder = os.path.join(self.export_folder, tmp_str) self.files_helper.check_created(data_export_folder) return data_export_folder def export_simulator_configuration(self, burst_id): burst = dao.get_burst_by_id(burst_id) if burst is None: raise InvalidExportDataException("Could not find burst with ID " + str(burst_id)) op_folder = self.files_helper.get_project_folder( burst.project, str(burst.fk_simulation)) tmp_export_folder = self._build_data_export_folder(burst) tmp_sim_folder = os.path.join(tmp_export_folder, self.EXPORTED_SIMULATION_NAME) if not os.path.exists(tmp_sim_folder): os.makedirs(tmp_sim_folder) all_view_model_paths, all_datatype_paths = h5.gather_references_of_view_model( burst.simulator_gid, op_folder) burst_path = h5.determine_filepath(burst.gid, op_folder) all_view_model_paths.append(burst_path) for vm_path in all_view_model_paths: dest = os.path.join(tmp_sim_folder, os.path.basename(vm_path)) self.files_helper.copy_file(vm_path, dest) for dt_path in all_datatype_paths: dest = os.path.join(tmp_sim_folder, self.EXPORTED_SIMULATION_DTS_DIR, os.path.basename(dt_path)) self.files_helper.copy_file(dt_path, dest) main_vm_path = h5.determine_filepath(burst.simulator_gid, tmp_sim_folder) H5File.remove_metadata_param(main_vm_path, 'history_gid') now = datetime.now() date_str = now.strftime("%Y-%m-%d_%H-%M") zip_file_name = "%s_%s.%s" % (date_str, str(burst_id), self.ZIP_FILE_EXTENSION) result_path = os.path.join(tmp_export_folder, zip_file_name) with TvbZip(result_path, "w") as zip_file: zip_file.write_folder(tmp_sim_folder) self.files_helper.remove_folder(tmp_sim_folder) return result_path
class FilesHelperTest(TransactionalTestCase): """ This class contains tests for the tvb.core.entities.file.files_helper module. """ PROJECT_NAME = "test_proj" def setUp(self): """ Set up the context needed by the tests. """ self.files_helper = FilesHelper() self.test_user = TestFactory.create_user() self.test_project = TestFactory.create_project(self.test_user, self.PROJECT_NAME) def tearDown(self): """ Remove generated project during tests. """ self.delete_project_folders() def test_check_created(self): """ Test standard flows for check created. """ self.files_helper.check_created() self.assertTrue(os.path.exists(root_storage), "Storage not created!") self.files_helper.check_created(os.path.join(root_storage, "test")) self.assertTrue(os.path.exists(root_storage), "Storage not created!") self.assertTrue(os.path.exists(os.path.join(root_storage, "test")), "Test directory not created!") def test_get_project_folder(self): """ Test the get_project_folder method which should create a folder in case it doesn't already exist. """ project_path = self.files_helper.get_project_folder(self.test_project) self.assertTrue(os.path.exists(project_path), "Folder doesn't exist") folder_path = self.files_helper.get_project_folder( self.test_project, "43") self.assertTrue(os.path.exists(project_path), "Folder doesn't exist") self.assertTrue(os.path.exists(folder_path), "Folder doesn't exist") def test_rename_project_structure(self): """ Try to rename the folder structure of a project. Standard flow. """ self.files_helper.get_project_folder(self.test_project) path, name = self.files_helper.rename_project_structure( self.test_project.name, "new_name") self.assertNotEqual(path, name, "Rename didn't take effect.") def test_rename_structure_same_name(self): """ Try to rename the folder structure of a project. Same name. """ self.files_helper.get_project_folder(self.test_project) self.assertRaises(FileStructureException, self.files_helper.rename_project_structure, self.test_project.name, self.PROJECT_NAME) def test_remove_project_structure(self): """ Check that remove project structure deletes the corresponding folder. Standard flow. """ full_path = self.files_helper.get_project_folder(self.test_project) self.assertTrue(os.path.exists(full_path), "Folder was not created.") self.files_helper.remove_project_structure(self.test_project.name) self.assertFalse(os.path.exists(full_path), "Project folder not deleted.") def test_write_project_metadata(self): """ Write XML for test-project. """ self.files_helper.write_project_metadata(self.test_project) expected_file = self.files_helper.get_project_meta_file_path( self.PROJECT_NAME) self.assertTrue(os.path.exists(expected_file)) project_meta = XMLReader(expected_file).read_metadata() loaded_project = model.Project(None, None) loaded_project.from_dict(project_meta, self.test_user.id) self.assertEqual(self.test_project.name, loaded_project.name) self.assertEqual(self.test_project.description, loaded_project.description) self.assertEqual(self.test_project.gid, loaded_project.gid) expected_dict = self.test_project.to_dict()[1] del expected_dict['last_updated'] found_dict = loaded_project.to_dict()[1] del found_dict['last_updated'] self.assertDictContainsSubset(expected_dict, found_dict) self.assertDictContainsSubset(found_dict, expected_dict) def test_write_operation_metadata(self): """ Test that a correct XML is created for an operation. """ operation = TestFactory.create_operation( test_user=self.test_user, test_project=self.test_project) expected_file = self.files_helper.get_operation_meta_file_path( self.PROJECT_NAME, operation.id) self.assertFalse(os.path.exists(expected_file)) self.files_helper.write_operation_metadata(operation) self.assertTrue(os.path.exists(expected_file)) operation_meta = XMLReader(expected_file).read_metadata() loaded_operation = model.Operation(None, None, None, None) loaded_operation.from_dict(operation_meta, dao) expected_dict = operation.to_dict()[1] found_dict = loaded_operation.to_dict()[1] for key, value in expected_dict.iteritems(): self.assertEqual(str(value), str(found_dict[key])) # Now validate that operation metaData can be also updated self.assertNotEqual("new_group_name", found_dict['user_group']) self.files_helper.update_operation_metadata(self.PROJECT_NAME, "new_group_name", operation.id) found_dict = XMLReader(expected_file).read_metadata() self.assertEqual("new_group_name", found_dict['user_group']) def test_remove_dt_happy_flow(self): """ Happy flow for removing a file related to a DataType. """ folder_path = self.files_helper.get_project_folder( self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path open(datatype.get_storage_file_path(), 'w') self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") self.files_helper.remove_datatype(datatype) self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not deleted!") def test_remove_dt_non_existent(self): """ Try to call remove on a dataType with no H5 file. Should work. """ folder_path = self.files_helper.get_project_folder( self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path self.assertFalse(os.path.exists(datatype.get_storage_file_path())) self.files_helper.remove_datatype(datatype) def test_move_datatype(self): """ Make sure associated H5 file is moved to a correct new location. """ folder_path = self.files_helper.get_project_folder( self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path open(datatype.get_storage_file_path(), 'w') self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") self.files_helper.move_datatype(datatype, self.PROJECT_NAME + '11', "43") self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not moved!") datatype.storage_path = self.files_helper.get_project_folder( self.PROJECT_NAME + '11', "43") self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") def test_find_relative_path(self): """ Tests that relative path is computed properly. """ rel_path = self.files_helper.find_relative_path( "/root/up/to/here/test/it/now", "/root/up/to/here") self.assertEqual(rel_path, os.sep.join(["test", "it", "now"]), "Did not extract relative path as expected.") def test_remove_files_valid(self): """ Pass a valid list of files and check they are all removed. """ file_list = ["test1", "test2", "test3"] for file_n in file_list: fp = open(file_n, 'w') fp.write('test') fp.close() for file_n in file_list: self.assertTrue(os.path.isfile(file_n)) self.files_helper.remove_files(file_list) for file_n in file_list: self.assertFalse(os.path.isfile(file_n)) def test_remove_folder(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" os.mkdir(folder_name) self.assertTrue(os.path.isdir(folder_name), "Folder should be created.") self.files_helper.remove_folder(folder_name) self.assertFalse(os.path.isdir(folder_name), "Folder should be deleted.") def test_remove_folder_non_existing_ignore_exc(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.") self.files_helper.remove_folder(folder_name, ignore_errors=True) def test_remove_folder_non_existing(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.") self.assertRaises(FileStructureException, self.files_helper.remove_folder, folder_name, False)