def delete(self, current_user_id: UserIDT, object_ids: ObjectIDListT) -> Tuple[int, int, int, int]:
"""
Remove from DB all the objects with ID in given list.
"""
# Security check
obj_set = EnumeratedObjectSet(self.session, object_ids)
# Get project IDs for the objects and verify rights
prj_ids = obj_set.get_projects_ids()
for a_prj_id in prj_ids:
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, a_prj_id)
# Prepare & start a remover thread that will run in // with DB queries
remover = VaultRemover(self.link_src, logger).do_start()
# Do the deletion itself.
nb_objs, nb_img_rows, img_files = obj_set.delete(self.CHUNK_SIZE, remover.add_files)
# Update stats on impacted project(s)
for prj_id in prj_ids:
ProjectBO.update_taxo_stats(self.session, prj_id)
# Stats depend on taxo stats
ProjectBO.update_stats(self.session, prj_id)
self.session.commit()
# Wait for the files handled
remover.wait_for_done()
return nb_objs, 0, nb_img_rows, len(img_files)
def do_run(self, current_user_id: int) -> MergeRsp:
"""
Run the service, merge the projects.
:return:
"""
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE,
self.prj_id)
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE,
self.src_prj_id)
# OK
prj = self.session.query(Project).get(self.prj_id)
assert prj is not None
src_prj = self.session.query(Project).get(self.src_prj_id)
assert src_prj is not None
logger.info("Validating Merge of '%s'", prj.title)
ret = MergeRsp()
errs = self._verify_possible(prj, src_prj)
ret.errors = errs
# Exit if errors or dry run
if self.dry_run or len(errs) > 0:
return ret
logger.info("Remaps: %s", self.remap_operations)
# Go for real if not dry run AND len(errs) == 0
logger.info("Starting Merge of '%s'", prj.title)
self._do_merge(prj)
self.session.commit()
# Recompute stats and so on
ProjectBO.do_after_load(self.session, prj_id=self.prj_id)
self.session.commit()
return ret
def run(self, current_user_id: int) -> ImportPrepRsp:
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE,
self.prj_id)
# OK
loaded_files = none_to_empty(self.prj.fileloaded).splitlines()
logger.info("Previously loaded files: %s", loaded_files)
self.manage_uploaded()
# Prepare response
ret = ImportPrepRsp(source_path=self.source_dir_or_zip)
self.update_progress(0, "Starting")
# Unzip or point to source directory
self.unzip_if_needed()
ret.source_path = self.source_dir_or_zip
# Validate files
logger.info("Analyze TSV Files")
how, diag, nb_rows = self.do_intra_step_1(loaded_files)
ret.mappings = how.custom_mapping.as_dict()
ret.warnings = diag.messages
ret.errors = diag.errors
ret.rowcount = nb_rows
# Resolve users...
logger.info("Resolve users")
self.resolve_users(self.session, how.found_users)
ret.found_users = how.found_users
# ...and taxonomy
logger.info("Resolve taxonomy")
self.resolve_taxa(self.session, how.taxo_found)
ret.found_taxa = how.taxo_found
return ret
def do_run(self, current_user_id: int) -> SubsetRsp:
# Security checks
RightsBO.user_wants(self.session, current_user_id, Action.READ, self.prj_id)
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, self.dest_prj.projid)
# OK
logger.info("Starting subset of '%s'", self.prj.title)
ret = SubsetRsp()
self.update_progress(5, "Determining objects to clone")
self._find_what_to_clone()
logger.info("Matched %s objects", len(self.to_clone))
if len(self.to_clone) == 0:
self.task.taskstate = "Error"
self.update_progress(10, "No object to include in the subset project")
ret.errors.append("No object found to clone into subset.")
return ret
self._do_clone()
self.session.commit()
# Recompute stats and so on
ProjectBO.do_after_load(self.session, self.dest_prj.projid)
self.session.commit()
return ret
def do_run(self, current_user_id: int) -> ImportRealRsp:
"""
Do the real job using injected parameters.
:return:
"""
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE,
self.prj_id)
# OK
loaded_files = none_to_empty(self.prj.fileloaded).splitlines()
logger.info("Previously loaded files: %s", loaded_files)
# Save mappings straight away
self.save_mapping(self.custom_mapping)
source_bundle = InBundle(
self.req.source_path,
Path(self.temp_for_task.data_dir_for(self.task_id)))
# Configure the import to come, destination
db_writer = DBWriter(self.session)
import_where = ImportWhere(
db_writer, self.vault,
self.temp_for_task.base_dir_for(self.task_id))
# Configure the import to come, directives
import_how = ImportHow(self.prj_id, self.req.update_mode,
self.custom_mapping,
self.req.skip_existing_objects, loaded_files)
import_how.taxo_mapping = self.req.taxo_mappings
import_how.taxo_found = self.req.found_taxa
import_how.found_users = self.req.found_users
if self.req.skip_loaded_files:
import_how.compute_skipped(source_bundle, logger)
if not self.req.skip_existing_objects:
with CodeTimer("run: Existing images for %d: " % self.prj_id,
logger):
import_how.objects_and_images_to_skip = Image.fetch_existing_images(
self.session, self.prj_id)
import_how.do_thumbnail_above(int(self.config['THUMBSIZELIMIT']))
# Do the bulk job of import
row_count = source_bundle.do_import(import_where, import_how,
self.req.rowcount,
self.report_progress)
# Update loaded files in DB, removing duplicates
self.prj.fileloaded = "\n".join(set(import_how.loaded_files))
self.session.commit()
# Recompute stats
ProjectBO.do_after_load(self.session, self.prj_id)
self.session.commit()
logger.info("Total of %d rows loaded" % row_count)
# Prepare response
ret = ImportRealRsp()
return ret
def do_run(self, current_user_id: int) -> List[str]:
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.READ,
self.prj_id)
# OK
ret = []
# TODO: Permissions
ret.extend(self.check_paths_unicity())
return ret
def search(self, current_user_id: Optional[UserIDT], project_id: ProjectIDT) -> List[AcquisitionBO]:
# Security check
if current_user_id is None:
project = RightsBO.anonymous_wants(self.ro_session, Action.READ, project_id)
else:
_user, project = RightsBO.user_wants(self.session, current_user_id, Action.READ, project_id)
acquisition_set = DescribedAcquisitionSet(self.ro_session, project_id)
# mappings = ProjectMapping().load_from_project(project)
# ret.map_free_columns(mappings.sample_mappings)
return acquisition_set.list()
def run(self, current_user_id: int) -> ImportRsp:
"""
Initial run, basically just create the job.
"""
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE,
self.prj_id)
# OK, go background straight away
self.create_job(self.JOB_TYPE, current_user_id)
ret = ImportRsp(job_id=self.job_id)
return ret
def run(self, current_user_id: int) -> Optional[SimpleImportRsp]:
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE,
self.prj_id)
# Validate values in all cases, dry run or not.
ret = self._validate()
if len(ret.errors) > 0:
return ret
if not self.dry_run:
self.create_job(self.JOB_TYPE, current_user_id)
ret.job_id = self.job_id
return ret
def query(self, current_user_id: Optional[int], acquisition_id: AcquisitionIDT) -> Optional[AcquisitionBO]:
ret = AcquisitionBO(self.ro_session, acquisition_id)
if not ret.exists():
return None
assert ret.acquis is not None
# Security check
if current_user_id is None:
project = RightsBO.anonymous_wants(self.ro_session, Action.READ, ret.acquis.sample.projid)
else:
_user, project = RightsBO.user_wants(self.session, current_user_id, Action.READ, ret.acquis.sample.projid)
mappings = ProjectMapping().load_from_project(project)
ret.map_free_columns(mappings.acquisition_mappings)
return ret
def read_taxo_stats(self, current_user_id: Optional[UserIDT],
sample_ids: SampleIDListT) -> List[SampleTaxoStats]:
# Get project IDs for the samples and verify rights
sample_set = EnumeratedSampleSet(self.ro_session, sample_ids)
project_ids = sample_set.get_projects_ids()
# Security check
if current_user_id is None:
[RightsBO.anonymous_wants(self.ro_session, Action.READ, project_id)
for project_id in project_ids]
else:
[RightsBO.user_wants(self.session, current_user_id, Action.READ, project_id)
for project_id in project_ids]
return sample_set.read_taxo_stats()
def do_run(self, current_user_id: int) -> SimpleImportRsp:
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE,
self.prj_id)
# OK
# Validate values in all cases
ret = self._validate()
if len(ret.errors) > 0:
return ret
if self.task_id != 0:
if len(ret.errors) == 0:
ret = self.do_import()
return ret
def query(self, current_user_id: Optional[UserIDT], sample_id: SampleIDT) -> Optional[SampleBO]:
ret = SampleBO(self.ro_session, sample_id)
if not ret.exists():
return None
assert ret.sample is not None
assert ret.sample.projid is not None # TODO: Why need this?
# Security check
if current_user_id is None:
project = RightsBO.anonymous_wants(self.ro_session, Action.READ, ret.sample.projid)
else:
_user, project = RightsBO.user_wants(self.session, current_user_id, Action.READ, ret.sample.projid)
mappings = ProjectMapping().load_from_project(project)
ret.map_free_columns(mappings.sample_mappings)
return ret
def search(self, current_user_id: Optional[UserIDT],
project_ids: ProjectIDListT,
orig_id_pattern: str) -> List[SampleBO]:
# Security check
if current_user_id is None:
[RightsBO.anonymous_wants(self.ro_session, Action.READ, project_id)
for project_id in project_ids]
else:
[RightsBO.user_wants(self.session, current_user_id, Action.READ, project_id)
for project_id in project_ids]
sample_set = DescribedSampleSet(self.ro_session, project_ids, orig_id_pattern)
# mappings = ProjectMapping().load_from_project(project)
# ret.map_free_columns(mappings.sample_mappings)
return sample_set.list()
def query(self, current_user_id: Optional[int], object_id: ObjectIDT) -> Optional[ObjectBO]:
ret = ObjectBO(self.ro_session, object_id)
if not ret.exists():
return None
# Security check
projid = ret.header.acquisition.sample.projid
if current_user_id is None:
project = RightsBO.anonymous_wants(self.session, Action.READ, projid)
else:
_user, project = RightsBO.user_wants(self.session, current_user_id, Action.READ, projid)
assert project is not None
mappings = ProjectMapping().load_from_project(project)
ret.map_free_columns(mappings.object_mappings)
return ret
def query(self, current_user_id: Optional[UserIDT], prj_id: int,
for_managing: bool) -> ProjectBO:
if current_user_id is None:
RightsBO.anonymous_wants(self.session, Action.READ, prj_id)
highest_right = ""
else:
current_user, project = RightsBO.user_wants(
self.session, current_user_id,
Action.ADMINISTRATE if for_managing else Action.READ, prj_id)
highest_right = RightsBO.highest_right_on(current_user, prj_id)
ret = ProjectBOSet.get_one(self.session, prj_id)
assert ret is not None
ret.highest_right = highest_right
return ret
def query_history(self, current_user_id: Optional[int], object_id: ObjectIDT) \
-> List[HistoricalClassification]:
the_obj = ObjectBO(self.ro_session, object_id)
if not the_obj.exists():
return []
# Security check
# TODO: dup code
projid = the_obj.header.acquisition.sample.projid
if current_user_id is None:
RightsBO.anonymous_wants(self.ro_session, Action.READ, projid)
else:
_user, project = RightsBO.user_wants(self.session, current_user_id, Action.READ, projid)
assert project is not None
ret = the_obj.get_history()
return ret
def query(self, current_user_id: UserIDT,
coll_id: CollectionIDT) -> Optional[CollectionBO]:
# TODO, for now only admins
_user = RightsBO.user_has_role(self.session, current_user_id,
Role.APP_ADMINISTRATOR)
ret = CollectionBO.get_one(self.session, coll_id)
return ret
def delete(self, current_user_id: int, prj_id: int,
only_objects: bool) -> Tuple[int, int, int, int]:
# Security barrier
_current_user, _project = RightsBO.user_wants(self.session,
current_user_id,
Action.ADMINISTRATE,
prj_id)
# Troll-ish way of erasing
all_object_ids = ProjectBO.get_all_object_ids(self.session,
prj_id=prj_id)
# Build a big set
obj_set = EnumeratedObjectSet(self.session, all_object_ids)
# Prepare a remover thread that will run in // with DB queries
remover = VaultRemover(self.link_src, logger).do_start()
# Do the deletion itself.
nb_objs, nb_img_rows, img_files = obj_set.delete(
self.DELETE_CHUNK_SIZE, remover.add_files)
ProjectBO.delete_object_parents(self.session, prj_id)
if only_objects:
# Update stats, should all be 0...
ProjectBO.update_taxo_stats(self.session, prj_id)
# Stats depend on taxo stats
ProjectBO.update_stats(self.session, prj_id)
else:
ProjectBO.delete(self.session, prj_id)
self.session.commit()
# Wait for the files handled
remover.wait_for_done()
return nb_objs, 0, nb_img_rows, len(img_files)
def recompute_geo(self, current_user_id: int, prj_id: ProjectIDT) -> None:
# Security barrier
_current_user, _project = RightsBO.user_wants(self.session,
current_user_id,
Action.ADMINISTRATE,
prj_id)
Sample.propagate_geo(self.session, prj_id)
def reset_to_predicted(self, current_user_id: UserIDT, proj_id: ProjectIDT, filters: ProjectFilters) -> None:
"""
Query the given project with given filters, reset the resulting objects to predicted.
"""
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, proj_id)
impacted_objs = [r[0] for r in self.query(current_user_id, proj_id, filters)[0]]
EnumeratedObjectSet(self.session, impacted_objs).reset_to_predicted()
# Update stats
ProjectBO.update_taxo_stats(self.session, proj_id)
# Stats depend on taxo stats
ProjectBO.update_stats(self.session, proj_id)
self.session.commit()
async def check_id(self, current_user_id, aphia_id) -> str:
"""
Check the given aphia_id, adjust the DB if needed.
"""
# Security check
_user = RightsBO.user_has_role(self.ro_session, current_user_id,
Role.APP_ADMINISTRATOR)
lineage = await WoRMSFinder.aphia_classif_by_id(aphia_id)
# Nested struct, e.g. : {'AphiaID': 1, 'rank': 'Superdomain', 'scientificname': 'Biota', 'child':
# {'AphiaID': 3, 'rank': 'Kingdom', 'scientificname': 'Plantae', 'child':
# {'AphiaID': 368663, 'rank': 'Subkingdom', 'scientificname': 'Viridiplantae', 'child':
# {'AphiaID': 536191, 'rank': 'Infrakingdom', 'scientificname': 'Streptophyta', 'child':
# ...
# }}}}}}}}}
prev_level = None
while lineage is not None:
aphia_id_for_level = lineage["AphiaID"]
db_entry = self.session.query(WoRMS).get(aphia_id_for_level)
if db_entry is None:
assert prev_level is not None
prev_level.all_fetched = False
self.session.commit()
return "%d was not found, so parent %d was marked as incomplete" % \
(aphia_id_for_level, prev_level.aphia_id)
lineage = lineage["child"]
prev_level = db_entry
return "All OK"
def do_run(self, current_user_id: int) -> List[str]:
# Security check
_user, project = RightsBO.user_wants(self.session, current_user_id,
Action.READ, self.prj_id)
# OK
proj_bo = ProjectBO(project).enrich()
ret = []
# TODO: Permissions
ret.append(proj_bo.title)
ret.append(str(proj_bo.obj_free_cols))
free_cols_vals = proj_bo.get_all_num_columns_values(self.session)
acquis_stats: AcquisitionStats = AcquisitionStats("", 0)
for a_row in free_cols_vals:
acquis_id, acquis_orig_id, objid, *free_vals = a_row
free_vals = [
a_val if a_val is not None else Decimal('nan')
for a_val in free_vals
]
if acquis_id == acquis_stats.acquis_id:
# Same acquisition
pass
else:
# New acquisition, close previous one
self.output_acq(acquis_stats, ret)
# And start new one
acquis_stats = AcquisitionStats(acquis_orig_id, acquis_id)
acquis_stats.add_values(free_vals)
self.output_acq(acquis_stats, ret)
return ret
def delete(self, current_user_id: UserIDT, coll_id: CollectionIDT) -> int:
# TODO, for now only admins
_user = RightsBO.user_has_role(self.ro_session, current_user_id,
Role.APP_ADMINISTRATOR)
CollectionBO.delete(self.session, coll_id)
self.session.commit()
return 0
async def db_refresh(self, current_user_id: int):
"""
Refresh the local taxonomy DB.
"""
# Security check
_user = RightsBO.user_has_role(self.ro_session, current_user_id,
Role.APP_ADMINISTRATOR)
await self._do_refresh()
def search(self, current_user_id: UserIDT,
title: str) -> List[CollectionBO]:
# TODO, for now only admins
_user = RightsBO.user_has_role(self.ro_session, current_user_id,
Role.APP_ADMINISTRATOR)
qry = self.ro_session.query(Collection).filter(
Collection.title.ilike(title))
ret = [CollectionBO(a_rec).enrich() for a_rec in qry.all()]
return ret
def create(self, current_user_id: UserIDT,
req: CreateCollectionReq) -> Union[CollectionIDT, str]:
"""
Create a collection.
"""
# TODO, for now only admins
_user = RightsBO.user_has_role(self.ro_session, current_user_id,
Role.APP_ADMINISTRATOR)
coll_id = CollectionBO.create(self.session, req.title, req.project_ids)
return coll_id
def run(self, current_user_id: int) -> ExportRsp:
"""
Initial run, basically just do security check and create the job.
"""
_user, _project = RightsBO.user_wants(self.session, current_user_id,
Action.READ, self.req.project_id)
# OK, go background straight away
self.create_job(self.JOB_TYPE, current_user_id)
ret = ExportRsp(job_id=self.job_id)
return ret
def update_set(self, current_user_id: UserIDT, sample_ids: SampleIDListT, updates: ColUpdateList):
# Get project IDs for the samples and verify rights
sample_set = EnumeratedSampleSet(self.session, sample_ids)
prj_ids = sample_set.get_projects_ids()
# All should be in same project, so far
assert len(prj_ids) == 1, "Too many or no projects for samples: %s" % sample_ids
prj_id = prj_ids[0]
_user, project = RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, prj_id)
assert project # for mypy
return sample_set.apply_on_all(project, updates)
def update_set(self, current_user_id: int, acquisition_ids: AcquisitionIDListT, updates: ColUpdateList):
# Get project IDs for the acquisitions and verify rights
acquisition_set = EnumeratedAcquisitionSet(self.session, acquisition_ids)
prj_ids = acquisition_set.get_projects_ids()
# All should be in same project, so far
assert len(prj_ids) == 1, "Too many or no projects for acquisitions: %s" % acquisition_ids
prj_id = prj_ids[0]
_user, project = RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, prj_id)
assert project # for mypy
return acquisition_set.apply_on_all(project, updates)
