def delete(self, current_user_id: UserIDT, object_ids: ObjectIDListT) -> Tuple[int, int, int, int]: """ Remove from DB all the objects with ID in given list. """ # Security check obj_set = EnumeratedObjectSet(self.session, object_ids) # Get project IDs for the objects and verify rights prj_ids = obj_set.get_projects_ids() for a_prj_id in prj_ids: RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, a_prj_id) # Prepare & start a remover thread that will run in // with DB queries remover = VaultRemover(self.link_src, logger).do_start() # Do the deletion itself. nb_objs, nb_img_rows, img_files = obj_set.delete(self.CHUNK_SIZE, remover.add_files) # Update stats on impacted project(s) for prj_id in prj_ids: ProjectBO.update_taxo_stats(self.session, prj_id) # Stats depend on taxo stats ProjectBO.update_stats(self.session, prj_id) self.session.commit() # Wait for the files handled remover.wait_for_done() return nb_objs, 0, nb_img_rows, len(img_files)
def do_run(self, current_user_id: int) -> MergeRsp: """ Run the service, merge the projects. :return: """ # Security check RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, self.prj_id) RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, self.src_prj_id) # OK prj = self.session.query(Project).get(self.prj_id) assert prj is not None src_prj = self.session.query(Project).get(self.src_prj_id) assert src_prj is not None logger.info("Validating Merge of '%s'", prj.title) ret = MergeRsp() errs = self._verify_possible(prj, src_prj) ret.errors = errs # Exit if errors or dry run if self.dry_run or len(errs) > 0: return ret logger.info("Remaps: %s", self.remap_operations) # Go for real if not dry run AND len(errs) == 0 logger.info("Starting Merge of '%s'", prj.title) self._do_merge(prj) self.session.commit() # Recompute stats and so on ProjectBO.do_after_load(self.session, prj_id=self.prj_id) self.session.commit() return ret
def run(self, current_user_id: int) -> ImportPrepRsp: # Security check RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, self.prj_id) # OK loaded_files = none_to_empty(self.prj.fileloaded).splitlines() logger.info("Previously loaded files: %s", loaded_files) self.manage_uploaded() # Prepare response ret = ImportPrepRsp(source_path=self.source_dir_or_zip) self.update_progress(0, "Starting") # Unzip or point to source directory self.unzip_if_needed() ret.source_path = self.source_dir_or_zip # Validate files logger.info("Analyze TSV Files") how, diag, nb_rows = self.do_intra_step_1(loaded_files) ret.mappings = how.custom_mapping.as_dict() ret.warnings = diag.messages ret.errors = diag.errors ret.rowcount = nb_rows # Resolve users... logger.info("Resolve users") self.resolve_users(self.session, how.found_users) ret.found_users = how.found_users # ...and taxonomy logger.info("Resolve taxonomy") self.resolve_taxa(self.session, how.taxo_found) ret.found_taxa = how.taxo_found return ret
def do_run(self, current_user_id: int) -> SubsetRsp: # Security checks RightsBO.user_wants(self.session, current_user_id, Action.READ, self.prj_id) RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, self.dest_prj.projid) # OK logger.info("Starting subset of '%s'", self.prj.title) ret = SubsetRsp() self.update_progress(5, "Determining objects to clone") self._find_what_to_clone() logger.info("Matched %s objects", len(self.to_clone)) if len(self.to_clone) == 0: self.task.taskstate = "Error" self.update_progress(10, "No object to include in the subset project") ret.errors.append("No object found to clone into subset.") return ret self._do_clone() self.session.commit() # Recompute stats and so on ProjectBO.do_after_load(self.session, self.dest_prj.projid) self.session.commit() return ret
def do_run(self, current_user_id: int) -> ImportRealRsp: """ Do the real job using injected parameters. :return: """ # Security check RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, self.prj_id) # OK loaded_files = none_to_empty(self.prj.fileloaded).splitlines() logger.info("Previously loaded files: %s", loaded_files) # Save mappings straight away self.save_mapping(self.custom_mapping) source_bundle = InBundle( self.req.source_path, Path(self.temp_for_task.data_dir_for(self.task_id))) # Configure the import to come, destination db_writer = DBWriter(self.session) import_where = ImportWhere( db_writer, self.vault, self.temp_for_task.base_dir_for(self.task_id)) # Configure the import to come, directives import_how = ImportHow(self.prj_id, self.req.update_mode, self.custom_mapping, self.req.skip_existing_objects, loaded_files) import_how.taxo_mapping = self.req.taxo_mappings import_how.taxo_found = self.req.found_taxa import_how.found_users = self.req.found_users if self.req.skip_loaded_files: import_how.compute_skipped(source_bundle, logger) if not self.req.skip_existing_objects: with CodeTimer("run: Existing images for %d: " % self.prj_id, logger): import_how.objects_and_images_to_skip = Image.fetch_existing_images( self.session, self.prj_id) import_how.do_thumbnail_above(int(self.config['THUMBSIZELIMIT'])) # Do the bulk job of import row_count = source_bundle.do_import(import_where, import_how, self.req.rowcount, self.report_progress) # Update loaded files in DB, removing duplicates self.prj.fileloaded = "\n".join(set(import_how.loaded_files)) self.session.commit() # Recompute stats ProjectBO.do_after_load(self.session, self.prj_id) self.session.commit() logger.info("Total of %d rows loaded" % row_count) # Prepare response ret = ImportRealRsp() return ret
def do_run(self, current_user_id: int) -> List[str]: # Security check RightsBO.user_wants(self.session, current_user_id, Action.READ, self.prj_id) # OK ret = [] # TODO: Permissions ret.extend(self.check_paths_unicity()) return ret
def search(self, current_user_id: Optional[UserIDT], project_id: ProjectIDT) -> List[AcquisitionBO]: # Security check if current_user_id is None: project = RightsBO.anonymous_wants(self.ro_session, Action.READ, project_id) else: _user, project = RightsBO.user_wants(self.session, current_user_id, Action.READ, project_id) acquisition_set = DescribedAcquisitionSet(self.ro_session, project_id) # mappings = ProjectMapping().load_from_project(project) # ret.map_free_columns(mappings.sample_mappings) return acquisition_set.list()
def run(self, current_user_id: int) -> ImportRsp: """ Initial run, basically just create the job. """ # Security check RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, self.prj_id) # OK, go background straight away self.create_job(self.JOB_TYPE, current_user_id) ret = ImportRsp(job_id=self.job_id) return ret
def run(self, current_user_id: int) -> Optional[SimpleImportRsp]: # Security check RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, self.prj_id) # Validate values in all cases, dry run or not. ret = self._validate() if len(ret.errors) > 0: return ret if not self.dry_run: self.create_job(self.JOB_TYPE, current_user_id) ret.job_id = self.job_id return ret
def query(self, current_user_id: Optional[int], acquisition_id: AcquisitionIDT) -> Optional[AcquisitionBO]: ret = AcquisitionBO(self.ro_session, acquisition_id) if not ret.exists(): return None assert ret.acquis is not None # Security check if current_user_id is None: project = RightsBO.anonymous_wants(self.ro_session, Action.READ, ret.acquis.sample.projid) else: _user, project = RightsBO.user_wants(self.session, current_user_id, Action.READ, ret.acquis.sample.projid) mappings = ProjectMapping().load_from_project(project) ret.map_free_columns(mappings.acquisition_mappings) return ret
def read_taxo_stats(self, current_user_id: Optional[UserIDT], sample_ids: SampleIDListT) -> List[SampleTaxoStats]: # Get project IDs for the samples and verify rights sample_set = EnumeratedSampleSet(self.ro_session, sample_ids) project_ids = sample_set.get_projects_ids() # Security check if current_user_id is None: [RightsBO.anonymous_wants(self.ro_session, Action.READ, project_id) for project_id in project_ids] else: [RightsBO.user_wants(self.session, current_user_id, Action.READ, project_id) for project_id in project_ids] return sample_set.read_taxo_stats()
def do_run(self, current_user_id: int) -> SimpleImportRsp: # Security check RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, self.prj_id) # OK # Validate values in all cases ret = self._validate() if len(ret.errors) > 0: return ret if self.task_id != 0: if len(ret.errors) == 0: ret = self.do_import() return ret
def query(self, current_user_id: Optional[UserIDT], sample_id: SampleIDT) -> Optional[SampleBO]: ret = SampleBO(self.ro_session, sample_id) if not ret.exists(): return None assert ret.sample is not None assert ret.sample.projid is not None # TODO: Why need this? # Security check if current_user_id is None: project = RightsBO.anonymous_wants(self.ro_session, Action.READ, ret.sample.projid) else: _user, project = RightsBO.user_wants(self.session, current_user_id, Action.READ, ret.sample.projid) mappings = ProjectMapping().load_from_project(project) ret.map_free_columns(mappings.sample_mappings) return ret
def search(self, current_user_id: Optional[UserIDT], project_ids: ProjectIDListT, orig_id_pattern: str) -> List[SampleBO]: # Security check if current_user_id is None: [RightsBO.anonymous_wants(self.ro_session, Action.READ, project_id) for project_id in project_ids] else: [RightsBO.user_wants(self.session, current_user_id, Action.READ, project_id) for project_id in project_ids] sample_set = DescribedSampleSet(self.ro_session, project_ids, orig_id_pattern) # mappings = ProjectMapping().load_from_project(project) # ret.map_free_columns(mappings.sample_mappings) return sample_set.list()
def query(self, current_user_id: Optional[int], object_id: ObjectIDT) -> Optional[ObjectBO]: ret = ObjectBO(self.ro_session, object_id) if not ret.exists(): return None # Security check projid = ret.header.acquisition.sample.projid if current_user_id is None: project = RightsBO.anonymous_wants(self.session, Action.READ, projid) else: _user, project = RightsBO.user_wants(self.session, current_user_id, Action.READ, projid) assert project is not None mappings = ProjectMapping().load_from_project(project) ret.map_free_columns(mappings.object_mappings) return ret
def query(self, current_user_id: Optional[UserIDT], prj_id: int, for_managing: bool) -> ProjectBO: if current_user_id is None: RightsBO.anonymous_wants(self.session, Action.READ, prj_id) highest_right = "" else: current_user, project = RightsBO.user_wants( self.session, current_user_id, Action.ADMINISTRATE if for_managing else Action.READ, prj_id) highest_right = RightsBO.highest_right_on(current_user, prj_id) ret = ProjectBOSet.get_one(self.session, prj_id) assert ret is not None ret.highest_right = highest_right return ret
def query_history(self, current_user_id: Optional[int], object_id: ObjectIDT) \ -> List[HistoricalClassification]: the_obj = ObjectBO(self.ro_session, object_id) if not the_obj.exists(): return [] # Security check # TODO: dup code projid = the_obj.header.acquisition.sample.projid if current_user_id is None: RightsBO.anonymous_wants(self.ro_session, Action.READ, projid) else: _user, project = RightsBO.user_wants(self.session, current_user_id, Action.READ, projid) assert project is not None ret = the_obj.get_history() return ret
def query(self, current_user_id: UserIDT, coll_id: CollectionIDT) -> Optional[CollectionBO]: # TODO, for now only admins _user = RightsBO.user_has_role(self.session, current_user_id, Role.APP_ADMINISTRATOR) ret = CollectionBO.get_one(self.session, coll_id) return ret
def delete(self, current_user_id: int, prj_id: int, only_objects: bool) -> Tuple[int, int, int, int]: # Security barrier _current_user, _project = RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, prj_id) # Troll-ish way of erasing all_object_ids = ProjectBO.get_all_object_ids(self.session, prj_id=prj_id) # Build a big set obj_set = EnumeratedObjectSet(self.session, all_object_ids) # Prepare a remover thread that will run in // with DB queries remover = VaultRemover(self.link_src, logger).do_start() # Do the deletion itself. nb_objs, nb_img_rows, img_files = obj_set.delete( self.DELETE_CHUNK_SIZE, remover.add_files) ProjectBO.delete_object_parents(self.session, prj_id) if only_objects: # Update stats, should all be 0... ProjectBO.update_taxo_stats(self.session, prj_id) # Stats depend on taxo stats ProjectBO.update_stats(self.session, prj_id) else: ProjectBO.delete(self.session, prj_id) self.session.commit() # Wait for the files handled remover.wait_for_done() return nb_objs, 0, nb_img_rows, len(img_files)
def recompute_geo(self, current_user_id: int, prj_id: ProjectIDT) -> None: # Security barrier _current_user, _project = RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, prj_id) Sample.propagate_geo(self.session, prj_id)
def reset_to_predicted(self, current_user_id: UserIDT, proj_id: ProjectIDT, filters: ProjectFilters) -> None: """ Query the given project with given filters, reset the resulting objects to predicted. """ # Security check RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, proj_id) impacted_objs = [r[0] for r in self.query(current_user_id, proj_id, filters)[0]] EnumeratedObjectSet(self.session, impacted_objs).reset_to_predicted() # Update stats ProjectBO.update_taxo_stats(self.session, proj_id) # Stats depend on taxo stats ProjectBO.update_stats(self.session, proj_id) self.session.commit()
async def check_id(self, current_user_id, aphia_id) -> str: """ Check the given aphia_id, adjust the DB if needed. """ # Security check _user = RightsBO.user_has_role(self.ro_session, current_user_id, Role.APP_ADMINISTRATOR) lineage = await WoRMSFinder.aphia_classif_by_id(aphia_id) # Nested struct, e.g. : {'AphiaID': 1, 'rank': 'Superdomain', 'scientificname': 'Biota', 'child': # {'AphiaID': 3, 'rank': 'Kingdom', 'scientificname': 'Plantae', 'child': # {'AphiaID': 368663, 'rank': 'Subkingdom', 'scientificname': 'Viridiplantae', 'child': # {'AphiaID': 536191, 'rank': 'Infrakingdom', 'scientificname': 'Streptophyta', 'child': # ... # }}}}}}}}} prev_level = None while lineage is not None: aphia_id_for_level = lineage["AphiaID"] db_entry = self.session.query(WoRMS).get(aphia_id_for_level) if db_entry is None: assert prev_level is not None prev_level.all_fetched = False self.session.commit() return "%d was not found, so parent %d was marked as incomplete" % \ (aphia_id_for_level, prev_level.aphia_id) lineage = lineage["child"] prev_level = db_entry return "All OK"
def do_run(self, current_user_id: int) -> List[str]: # Security check _user, project = RightsBO.user_wants(self.session, current_user_id, Action.READ, self.prj_id) # OK proj_bo = ProjectBO(project).enrich() ret = [] # TODO: Permissions ret.append(proj_bo.title) ret.append(str(proj_bo.obj_free_cols)) free_cols_vals = proj_bo.get_all_num_columns_values(self.session) acquis_stats: AcquisitionStats = AcquisitionStats("", 0) for a_row in free_cols_vals: acquis_id, acquis_orig_id, objid, *free_vals = a_row free_vals = [ a_val if a_val is not None else Decimal('nan') for a_val in free_vals ] if acquis_id == acquis_stats.acquis_id: # Same acquisition pass else: # New acquisition, close previous one self.output_acq(acquis_stats, ret) # And start new one acquis_stats = AcquisitionStats(acquis_orig_id, acquis_id) acquis_stats.add_values(free_vals) self.output_acq(acquis_stats, ret) return ret
def delete(self, current_user_id: UserIDT, coll_id: CollectionIDT) -> int: # TODO, for now only admins _user = RightsBO.user_has_role(self.ro_session, current_user_id, Role.APP_ADMINISTRATOR) CollectionBO.delete(self.session, coll_id) self.session.commit() return 0
async def db_refresh(self, current_user_id: int): """ Refresh the local taxonomy DB. """ # Security check _user = RightsBO.user_has_role(self.ro_session, current_user_id, Role.APP_ADMINISTRATOR) await self._do_refresh()
def search(self, current_user_id: UserIDT, title: str) -> List[CollectionBO]: # TODO, for now only admins _user = RightsBO.user_has_role(self.ro_session, current_user_id, Role.APP_ADMINISTRATOR) qry = self.ro_session.query(Collection).filter( Collection.title.ilike(title)) ret = [CollectionBO(a_rec).enrich() for a_rec in qry.all()] return ret
def create(self, current_user_id: UserIDT, req: CreateCollectionReq) -> Union[CollectionIDT, str]: """ Create a collection. """ # TODO, for now only admins _user = RightsBO.user_has_role(self.ro_session, current_user_id, Role.APP_ADMINISTRATOR) coll_id = CollectionBO.create(self.session, req.title, req.project_ids) return coll_id
def run(self, current_user_id: int) -> ExportRsp: """ Initial run, basically just do security check and create the job. """ _user, _project = RightsBO.user_wants(self.session, current_user_id, Action.READ, self.req.project_id) # OK, go background straight away self.create_job(self.JOB_TYPE, current_user_id) ret = ExportRsp(job_id=self.job_id) return ret
def update_set(self, current_user_id: UserIDT, sample_ids: SampleIDListT, updates: ColUpdateList): # Get project IDs for the samples and verify rights sample_set = EnumeratedSampleSet(self.session, sample_ids) prj_ids = sample_set.get_projects_ids() # All should be in same project, so far assert len(prj_ids) == 1, "Too many or no projects for samples: %s" % sample_ids prj_id = prj_ids[0] _user, project = RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, prj_id) assert project # for mypy return sample_set.apply_on_all(project, updates)
def update_set(self, current_user_id: int, acquisition_ids: AcquisitionIDListT, updates: ColUpdateList): # Get project IDs for the acquisitions and verify rights acquisition_set = EnumeratedAcquisitionSet(self.session, acquisition_ids) prj_ids = acquisition_set.get_projects_ids() # All should be in same project, so far assert len(prj_ids) == 1, "Too many or no projects for acquisitions: %s" % acquisition_ids prj_id = prj_ids[0] _user, project = RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, prj_id) assert project # for mypy return acquisition_set.apply_on_all(project, updates)