def run(self, current_user_id: int) -> ImportPrepRsp:
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE,
self.prj_id)
# OK
loaded_files = none_to_empty(self.prj.fileloaded).splitlines()
logger.info("Previously loaded files: %s", loaded_files)
self.manage_uploaded()
# Prepare response
ret = ImportPrepRsp(source_path=self.source_dir_or_zip)
self.update_progress(0, "Starting")
# Unzip or point to source directory
self.unzip_if_needed()
ret.source_path = self.source_dir_or_zip
# Validate files
logger.info("Analyze TSV Files")
how, diag, nb_rows = self.do_intra_step_1(loaded_files)
ret.mappings = how.custom_mapping.as_dict()
ret.warnings = diag.messages
ret.errors = diag.errors
ret.rowcount = nb_rows
# Resolve users...
logger.info("Resolve users")
self.resolve_users(self.session, how.found_users)
ret.found_users = how.found_users
# ...and taxonomy
logger.info("Resolve taxonomy")
self.resolve_taxa(self.session, how.taxo_found)
ret.found_taxa = how.taxo_found
return ret
def do_run(self, current_user_id: int) -> SubsetRsp:
# Security checks
RightsBO.user_wants(self.session, current_user_id, Action.READ, self.prj_id)
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, self.dest_prj.projid)
# OK
logger.info("Starting subset of '%s'", self.prj.title)
ret = SubsetRsp()
self.update_progress(5, "Determining objects to clone")
self._find_what_to_clone()
logger.info("Matched %s objects", len(self.to_clone))
if len(self.to_clone) == 0:
self.task.taskstate = "Error"
self.update_progress(10, "No object to include in the subset project")
ret.errors.append("No object found to clone into subset.")
return ret
self._do_clone()
self.session.commit()
# Recompute stats and so on
ProjectBO.do_after_load(self.session, self.dest_prj.projid)
self.session.commit()
return ret
def do_run(self, current_user_id: int) -> MergeRsp:
"""
Run the service, merge the projects.
:return:
"""
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE,
self.prj_id)
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE,
self.src_prj_id)
# OK
prj = self.session.query(Project).get(self.prj_id)
assert prj is not None
src_prj = self.session.query(Project).get(self.src_prj_id)
assert src_prj is not None
logger.info("Validating Merge of '%s'", prj.title)
ret = MergeRsp()
errs = self._verify_possible(prj, src_prj)
ret.errors = errs
# Exit if errors or dry run
if self.dry_run or len(errs) > 0:
return ret
logger.info("Remaps: %s", self.remap_operations)
# Go for real if not dry run AND len(errs) == 0
logger.info("Starting Merge of '%s'", prj.title)
self._do_merge(prj)
self.session.commit()
# Recompute stats and so on
ProjectBO.do_after_load(self.session, prj_id=self.prj_id)
self.session.commit()
return ret
def delete(self, current_user_id: UserIDT, object_ids: ObjectIDListT) -> Tuple[int, int, int, int]:
"""
Remove from DB all the objects with ID in given list.
"""
# Security check
obj_set = EnumeratedObjectSet(self.session, object_ids)
# Get project IDs for the objects and verify rights
prj_ids = obj_set.get_projects_ids()
for a_prj_id in prj_ids:
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, a_prj_id)
# Prepare & start a remover thread that will run in // with DB queries
remover = VaultRemover(self.link_src, logger).do_start()
# Do the deletion itself.
nb_objs, nb_img_rows, img_files = obj_set.delete(self.CHUNK_SIZE, remover.add_files)
# Update stats on impacted project(s)
for prj_id in prj_ids:
ProjectBO.update_taxo_stats(self.session, prj_id)
# Stats depend on taxo stats
ProjectBO.update_stats(self.session, prj_id)
self.session.commit()
# Wait for the files handled
remover.wait_for_done()
return nb_objs, 0, nb_img_rows, len(img_files)
def do_run(self, current_user_id: int) -> ImportRealRsp:
"""
Do the real job using injected parameters.
:return:
"""
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE,
self.prj_id)
# OK
loaded_files = none_to_empty(self.prj.fileloaded).splitlines()
logger.info("Previously loaded files: %s", loaded_files)
# Save mappings straight away
self.save_mapping(self.custom_mapping)
source_bundle = InBundle(
self.req.source_path,
Path(self.temp_for_task.data_dir_for(self.task_id)))
# Configure the import to come, destination
db_writer = DBWriter(self.session)
import_where = ImportWhere(
db_writer, self.vault,
self.temp_for_task.base_dir_for(self.task_id))
# Configure the import to come, directives
import_how = ImportHow(self.prj_id, self.req.update_mode,
self.custom_mapping,
self.req.skip_existing_objects, loaded_files)
import_how.taxo_mapping = self.req.taxo_mappings
import_how.taxo_found = self.req.found_taxa
import_how.found_users = self.req.found_users
if self.req.skip_loaded_files:
import_how.compute_skipped(source_bundle, logger)
if not self.req.skip_existing_objects:
with CodeTimer("run: Existing images for %d: " % self.prj_id,
logger):
import_how.objects_and_images_to_skip = Image.fetch_existing_images(
self.session, self.prj_id)
import_how.do_thumbnail_above(int(self.config['THUMBSIZELIMIT']))
# Do the bulk job of import
row_count = source_bundle.do_import(import_where, import_how,
self.req.rowcount,
self.report_progress)
# Update loaded files in DB, removing duplicates
self.prj.fileloaded = "\n".join(set(import_how.loaded_files))
self.session.commit()
# Recompute stats
ProjectBO.do_after_load(self.session, self.prj_id)
self.session.commit()
logger.info("Total of %d rows loaded" % row_count)
# Prepare response
ret = ImportRealRsp()
return ret
def do_run(self, current_user_id: int) -> List[str]:
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.READ,
self.prj_id)
# OK
ret = []
# TODO: Permissions
ret.extend(self.check_paths_unicity())
return ret
def run(self, current_user_id: int) -> ImportRsp:
"""
Initial run, basically just create the job.
"""
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE,
self.prj_id)
# OK, go background straight away
self.create_job(self.JOB_TYPE, current_user_id)
ret = ImportRsp(job_id=self.job_id)
return ret
def run(self, current_user_id: int) -> Optional[SimpleImportRsp]:
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE,
self.prj_id)
# Validate values in all cases, dry run or not.
ret = self._validate()
if len(ret.errors) > 0:
return ret
if not self.dry_run:
self.create_job(self.JOB_TYPE, current_user_id)
ret.job_id = self.job_id
return ret
def do_run(self, current_user_id: int) -> SimpleImportRsp:
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE,
self.prj_id)
# OK
# Validate values in all cases
ret = self._validate()
if len(ret.errors) > 0:
return ret
if self.task_id != 0:
if len(ret.errors) == 0:
ret = self.do_import()
return ret
def do_run(self, current_user_id: int) -> List[str]:
# Security check
_user, project = RightsBO.user_wants(self.session, current_user_id,
Action.READ, self.prj_id)
# OK
proj_bo = ProjectBO(project).enrich()
ret = []
# TODO: Permissions
ret.append(proj_bo.title)
ret.append(str(proj_bo.obj_free_cols))
free_cols_vals = proj_bo.get_all_num_columns_values(self.session)
acquis_stats: AcquisitionStats = AcquisitionStats("", 0)
for a_row in free_cols_vals:
acquis_id, acquis_orig_id, objid, *free_vals = a_row
free_vals = [
a_val if a_val is not None else Decimal('nan')
for a_val in free_vals
]
if acquis_id == acquis_stats.acquis_id:
# Same acquisition
pass
else:
# New acquisition, close previous one
self.output_acq(acquis_stats, ret)
# And start new one
acquis_stats = AcquisitionStats(acquis_orig_id, acquis_id)
acquis_stats.add_values(free_vals)
self.output_acq(acquis_stats, ret)
return ret
def reset_to_predicted(self, current_user_id: UserIDT, proj_id: ProjectIDT, filters: ProjectFilters) -> None:
"""
Query the given project with given filters, reset the resulting objects to predicted.
"""
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, proj_id)
impacted_objs = [r[0] for r in self.query(current_user_id, proj_id, filters)[0]]
EnumeratedObjectSet(self.session, impacted_objs).reset_to_predicted()
# Update stats
ProjectBO.update_taxo_stats(self.session, proj_id)
# Stats depend on taxo stats
ProjectBO.update_stats(self.session, proj_id)
self.session.commit()
def recompute_geo(self, current_user_id: int, prj_id: ProjectIDT) -> None:
# Security barrier
_current_user, _project = RightsBO.user_wants(self.session,
current_user_id,
Action.ADMINISTRATE,
prj_id)
Sample.propagate_geo(self.session, prj_id)
def delete(self, current_user_id: int, prj_id: int,
only_objects: bool) -> Tuple[int, int, int, int]:
# Security barrier
_current_user, _project = RightsBO.user_wants(self.session,
current_user_id,
Action.ADMINISTRATE,
prj_id)
# Troll-ish way of erasing
all_object_ids = ProjectBO.get_all_object_ids(self.session,
prj_id=prj_id)
# Build a big set
obj_set = EnumeratedObjectSet(self.session, all_object_ids)
# Prepare a remover thread that will run in // with DB queries
remover = VaultRemover(self.link_src, logger).do_start()
# Do the deletion itself.
nb_objs, nb_img_rows, img_files = obj_set.delete(
self.DELETE_CHUNK_SIZE, remover.add_files)
ProjectBO.delete_object_parents(self.session, prj_id)
if only_objects:
# Update stats, should all be 0...
ProjectBO.update_taxo_stats(self.session, prj_id)
# Stats depend on taxo stats
ProjectBO.update_stats(self.session, prj_id)
else:
ProjectBO.delete(self.session, prj_id)
self.session.commit()
# Wait for the files handled
remover.wait_for_done()
return nb_objs, 0, nb_img_rows, len(img_files)
def read_user_stats(self, current_user_id: int,
prj_ids: ProjectIDListT) -> List[ProjectUserStats]:
"""
Read user statistics for these projects.
"""
# Security barrier
[RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, prj_id)
for prj_id in prj_ids]
ret = ProjectBO.read_user_stats(self.session, prj_ids)
return ret
def update_set(self, current_user_id: int, acquisition_ids: AcquisitionIDListT, updates: ColUpdateList):
# Get project IDs for the acquisitions and verify rights
acquisition_set = EnumeratedAcquisitionSet(self.session, acquisition_ids)
prj_ids = acquisition_set.get_projects_ids()
# All should be in same project, so far
assert len(prj_ids) == 1, "Too many or no projects for acquisitions: %s" % acquisition_ids
prj_id = prj_ids[0]
_user, project = RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, prj_id)
assert project # for mypy
return acquisition_set.apply_on_all(project, updates)
def update_set(self, current_user_id: UserIDT, sample_ids: SampleIDListT, updates: ColUpdateList):
# Get project IDs for the samples and verify rights
sample_set = EnumeratedSampleSet(self.session, sample_ids)
prj_ids = sample_set.get_projects_ids()
# All should be in same project, so far
assert len(prj_ids) == 1, "Too many or no projects for samples: %s" % sample_ids
prj_id = prj_ids[0]
_user, project = RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, prj_id)
assert project # for mypy
return sample_set.apply_on_all(project, updates)
def search(self, current_user_id: Optional[UserIDT], project_id: ProjectIDT) -> List[AcquisitionBO]:
# Security check
if current_user_id is None:
project = RightsBO.anonymous_wants(self.ro_session, Action.READ, project_id)
else:
_user, project = RightsBO.user_wants(self.session, current_user_id, Action.READ, project_id)
acquisition_set = DescribedAcquisitionSet(self.ro_session, project_id)
# mappings = ProjectMapping().load_from_project(project)
# ret.map_free_columns(mappings.sample_mappings)
return acquisition_set.list()
def run(self, current_user_id: int) -> ExportRsp:
"""
Initial run, basically just do security check and create the job.
"""
_user, _project = RightsBO.user_wants(self.session, current_user_id,
Action.READ, self.req.project_id)
# OK, go background straight away
self.create_job(self.JOB_TYPE, current_user_id)
ret = ExportRsp(job_id=self.job_id)
return ret
def query(self, current_user_id: Optional[int], acquisition_id: AcquisitionIDT) -> Optional[AcquisitionBO]:
ret = AcquisitionBO(self.ro_session, acquisition_id)
if not ret.exists():
return None
assert ret.acquis is not None
# Security check
if current_user_id is None:
project = RightsBO.anonymous_wants(self.ro_session, Action.READ, ret.acquis.sample.projid)
else:
_user, project = RightsBO.user_wants(self.session, current_user_id, Action.READ, ret.acquis.sample.projid)
mappings = ProjectMapping().load_from_project(project)
ret.map_free_columns(mappings.acquisition_mappings)
return ret
def read_taxo_stats(self, current_user_id: Optional[UserIDT],
sample_ids: SampleIDListT) -> List[SampleTaxoStats]:
# Get project IDs for the samples and verify rights
sample_set = EnumeratedSampleSet(self.ro_session, sample_ids)
project_ids = sample_set.get_projects_ids()
# Security check
if current_user_id is None:
[RightsBO.anonymous_wants(self.ro_session, Action.READ, project_id)
for project_id in project_ids]
else:
[RightsBO.user_wants(self.session, current_user_id, Action.READ, project_id)
for project_id in project_ids]
return sample_set.read_taxo_stats()
def search(self, current_user_id: Optional[UserIDT],
project_ids: ProjectIDListT,
orig_id_pattern: str) -> List[SampleBO]:
# Security check
if current_user_id is None:
[RightsBO.anonymous_wants(self.ro_session, Action.READ, project_id)
for project_id in project_ids]
else:
[RightsBO.user_wants(self.session, current_user_id, Action.READ, project_id)
for project_id in project_ids]
sample_set = DescribedSampleSet(self.ro_session, project_ids, orig_id_pattern)
# mappings = ProjectMapping().load_from_project(project)
# ret.map_free_columns(mappings.sample_mappings)
return sample_set.list()
def query(self, current_user_id: Optional[UserIDT], sample_id: SampleIDT) -> Optional[SampleBO]:
ret = SampleBO(self.ro_session, sample_id)
if not ret.exists():
return None
assert ret.sample is not None
assert ret.sample.projid is not None # TODO: Why need this?
# Security check
if current_user_id is None:
project = RightsBO.anonymous_wants(self.ro_session, Action.READ, ret.sample.projid)
else:
_user, project = RightsBO.user_wants(self.session, current_user_id, Action.READ, ret.sample.projid)
mappings = ProjectMapping().load_from_project(project)
ret.map_free_columns(mappings.sample_mappings)
return ret
def query(self, current_user_id: Optional[int], object_id: ObjectIDT) -> Optional[ObjectBO]:
ret = ObjectBO(self.ro_session, object_id)
if not ret.exists():
return None
# Security check
projid = ret.header.acquisition.sample.projid
if current_user_id is None:
project = RightsBO.anonymous_wants(self.session, Action.READ, projid)
else:
_user, project = RightsBO.user_wants(self.session, current_user_id, Action.READ, projid)
assert project is not None
mappings = ProjectMapping().load_from_project(project)
ret.map_free_columns(mappings.object_mappings)
return ret
def query(self, current_user_id: Optional[UserIDT], prj_id: int,
for_managing: bool) -> ProjectBO:
if current_user_id is None:
RightsBO.anonymous_wants(self.session, Action.READ, prj_id)
highest_right = ""
else:
current_user, project = RightsBO.user_wants(
self.session, current_user_id,
Action.ADMINISTRATE if for_managing else Action.READ, prj_id)
highest_right = RightsBO.highest_right_on(current_user, prj_id)
ret = ProjectBOSet.get_one(self.session, prj_id)
assert ret is not None
ret.highest_right = highest_right
return ret
def parents_by_id(self, current_user_id: UserIDT, object_ids: ObjectIDListT) -> ObjectIDWithParentsListT:
"""
Query the given IDs, return parents.
"""
# Security check
obj_set = EnumeratedObjectSet(self.session, object_ids)
# Get project IDs for the objects and verify rights
prj_ids = obj_set.get_projects_ids()
for a_prj_id in prj_ids:
RightsBO.user_wants(self.session, current_user_id, Action.READ, a_prj_id)
sql = """
SELECT obh.objid, acq.acquisid, sam.sampleid, sam.projid
FROM obj_head obh
JOIN acquisitions acq on acq.acquisid = obh.acquisid
JOIN samples sam on sam.sampleid = acq.acq_sample_id
WHERE obh.objid = any (:ids) """
params = {"ids": object_ids}
res: ResultProxy = self.session.execute(sql, params)
ids = [(objid, acquisid, sampleid, projid)
for objid, acquisid, sampleid, projid in res]
return ids # type:ignore
def _the_project_for(self, current_user_id: UserIDT, target_ids: ObjectIDListT, action: Action) \
-> Tuple[EnumeratedObjectSet, Project]:
"""
Check _the_ single project for an object set, with the given right.
"""
# Get project IDs for the objects and verify rights
object_set = EnumeratedObjectSet(self.session, target_ids)
prj_ids = object_set.get_projects_ids()
# All should be in same project, so far
assert len(prj_ids) == 1, "Too many or no projects for objects: %s" % target_ids
prj_id = prj_ids[0]
_user, project = RightsBO.user_wants(self.session, current_user_id, action, prj_id)
assert project # for mypy
return object_set, project
def revert_to_history(self, current_user_id: UserIDT, proj_id: ProjectIDT,
filters: ProjectFilters, dry_run: bool,
target: Optional[int]) -> Tuple[List[HistoricalLastClassif], ClassifSetInfoT]:
"""
Revert to classification history the given set, if dry_run then only simulate.
"""
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, proj_id)
# Get target objects
impacted_objs = [r[0] for r in self.query(current_user_id, proj_id, filters)[0]]
obj_set = EnumeratedObjectSet(self.session, impacted_objs)
# We don't revert to a previous version in history from same annotator
but_not_by: Optional[int] = None
but_not_by_str = filters.get('filt_last_annot', None)
if but_not_by_str is not None:
try:
but_not_by = int(but_not_by_str)
except ValueError:
pass
if dry_run:
# Return information on what to do
impact = obj_set.evaluate_revert_to_history(target, but_not_by)
# And names for display
classifs = TaxonomyBO.names_with_parent_for(self.session, self.collect_classif(impact))
else:
# Do the real thing
impact = obj_set.revert_to_history(target, but_not_by)
classifs = {}
# Update stats
ProjectBO.update_taxo_stats(self.session, proj_id)
# Stats depend on taxo stats
ProjectBO.update_stats(self.session, proj_id)
self.session.commit()
# Give feedback
return impact, classifs
def query_history(self, current_user_id: Optional[int], object_id: ObjectIDT) \
-> List[HistoricalClassification]:
the_obj = ObjectBO(self.ro_session, object_id)
if not the_obj.exists():
return []
# Security check
# TODO: dup code
projid = the_obj.header.acquisition.sample.projid
if current_user_id is None:
RightsBO.anonymous_wants(self.ro_session, Action.READ, projid)
else:
_user, project = RightsBO.user_wants(self.session, current_user_id, Action.READ, projid)
assert project is not None
ret = the_obj.get_history()
return ret
def summary(self, current_user_id: Optional[UserIDT], proj_id: ProjectIDT, filters: ProjectFilters,
only_total: bool) -> Tuple[int, Optional[int], Optional[int], Optional[int]]:
"""
Query the given project with given filters, return classification summary, or just grand total if
only_total is set.
"""
# Security check
if current_user_id is None:
RightsBO.anonymous_wants(self.session, Action.READ, proj_id)
# Anonymous can only see validated objects
# TODO: Dup code
# noinspection PyTypeHints
filters.statusfilter = "V" # type:ignore
user_id = -1
else:
user, _project = RightsBO.user_wants(self.session, current_user_id, Action.READ, proj_id)
user_id = user.id
# Prepare a where clause and parameters from filter
object_set: DescribedObjectSet = DescribedObjectSet(self.session, proj_id, filters)
from_, where, params = object_set.get_sql(user_id)
sql = """
SET LOCAL enable_seqscan=FALSE;
SELECT COUNT(*) nbr"""
if only_total:
sql += """, NULL nbr_v, NULL nbr_d, NULL nbr_p"""
else:
sql += """,
COUNT(CASE WHEN obh.classif_qual = 'V' THEN 1 END) nbr_v,
COUNT(CASE WHEN obh.classif_qual = 'D' THEN 1 END) nbr_d,
COUNT(CASE WHEN obh.classif_qual = 'P' THEN 1 END) nbr_p"""
sql += """
FROM """ + from_.get_sql() + " " + where.get_sql()
with CodeTimer("summary: V/D/P for %d using %s " % (proj_id, sql), logger):
res: ResultProxy = self.session.execute(sql, params)
nbr: int
nbr_v: Optional[int]
nbr_d: Optional[int]
nbr_p: Optional[int]
nbr, nbr_v, nbr_d, nbr_p = res.first() # type:ignore
return nbr, nbr_v, nbr_d, nbr_p
def create(self, current_user_id: int,
req: CreateProjectReq) -> Union[int, str]:
"""
Create a project, eventually as a shallow copy of another.
"""
if req.clone_of_id:
# Cloning a project needs only manager rights on the origin
current_user, prj = RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, req.clone_of_id)
if prj is None:
return "Project to clone not found"
prj = clone_of(prj)
else:
current_user = RightsBO.user_wants_create_project(self.session, current_user_id)
prj = Project()
prj.title = req.title
prj.status = ANNOTATE_STATUS
prj.visible = req.visible
self.session.add(prj)
self.session.flush() # to get the project ID
# Add the manage privilege
RightsBO.grant(self.session, current_user, Action.ADMINISTRATE, prj)
self.session.commit()
return prj.projid