def do_run(self, current_user_id: int) -> MergeRsp:
"""
Run the service, merge the projects.
:return:
"""
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE,
self.prj_id)
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE,
self.src_prj_id)
# OK
prj = self.session.query(Project).get(self.prj_id)
assert prj is not None
src_prj = self.session.query(Project).get(self.src_prj_id)
assert src_prj is not None
logger.info("Validating Merge of '%s'", prj.title)
ret = MergeRsp()
errs = self._verify_possible(prj, src_prj)
ret.errors = errs
# Exit if errors or dry run
if self.dry_run or len(errs) > 0:
return ret
logger.info("Remaps: %s", self.remap_operations)
# Go for real if not dry run AND len(errs) == 0
logger.info("Starting Merge of '%s'", prj.title)
self._do_merge(prj)
self.session.commit()
# Recompute stats and so on
ProjectBO.do_after_load(self.session, prj_id=self.prj_id)
self.session.commit()
return ret
def do_run(self, current_user_id: int) -> SubsetRsp:
# Security checks
RightsBO.user_wants(self.session, current_user_id, Action.READ, self.prj_id)
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE, self.dest_prj.projid)
# OK
logger.info("Starting subset of '%s'", self.prj.title)
ret = SubsetRsp()
self.update_progress(5, "Determining objects to clone")
self._find_what_to_clone()
logger.info("Matched %s objects", len(self.to_clone))
if len(self.to_clone) == 0:
self.task.taskstate = "Error"
self.update_progress(10, "No object to include in the subset project")
ret.errors.append("No object found to clone into subset.")
return ret
self._do_clone()
self.session.commit()
# Recompute stats and so on
ProjectBO.do_after_load(self.session, self.dest_prj.projid)
self.session.commit()
return ret
def do_run(self, current_user_id: int) -> ImportRealRsp:
"""
Do the real job using injected parameters.
:return:
"""
# Security check
RightsBO.user_wants(self.session, current_user_id, Action.ADMINISTRATE,
self.prj_id)
# OK
loaded_files = none_to_empty(self.prj.fileloaded).splitlines()
logger.info("Previously loaded files: %s", loaded_files)
# Save mappings straight away
self.save_mapping(self.custom_mapping)
source_bundle = InBundle(
self.req.source_path,
Path(self.temp_for_task.data_dir_for(self.task_id)))
# Configure the import to come, destination
db_writer = DBWriter(self.session)
import_where = ImportWhere(
db_writer, self.vault,
self.temp_for_task.base_dir_for(self.task_id))
# Configure the import to come, directives
import_how = ImportHow(self.prj_id, self.req.update_mode,
self.custom_mapping,
self.req.skip_existing_objects, loaded_files)
import_how.taxo_mapping = self.req.taxo_mappings
import_how.taxo_found = self.req.found_taxa
import_how.found_users = self.req.found_users
if self.req.skip_loaded_files:
import_how.compute_skipped(source_bundle, logger)
if not self.req.skip_existing_objects:
with CodeTimer("run: Existing images for %d: " % self.prj_id,
logger):
import_how.objects_and_images_to_skip = Image.fetch_existing_images(
self.session, self.prj_id)
import_how.do_thumbnail_above(int(self.config['THUMBSIZELIMIT']))
# Do the bulk job of import
row_count = source_bundle.do_import(import_where, import_how,
self.req.rowcount,
self.report_progress)
# Update loaded files in DB, removing duplicates
self.prj.fileloaded = "\n".join(set(import_how.loaded_files))
self.session.commit()
# Recompute stats
ProjectBO.do_after_load(self.session, self.prj_id)
self.session.commit()
logger.info("Total of %d rows loaded" % row_count)
# Prepare response
ret = ImportRealRsp()
return ret
def do_real(self) -> None:
"""
Do the real job, i.e. write everywhere (DB/filesystem)
"""
loaded_files = none_to_empty(self.prj.fileloaded).splitlines()
logger.info("Previously loaded files: %s", loaded_files)
found_users, taxo_found, col_mapping_dict, \
nb_rows, source_path = self._load_vars_from_state(self.STATE_KEYS)
# Save mappings straight away
col_mapping = ProjectMapping().load_from_dict(col_mapping_dict)
col_mapping.write_to_project(self.prj)
self.session.commit()
# TODO: Duplicated code
source_bundle = InBundle(
source_path, Path(self.temp_for_jobs.data_dir_for(self.job_id)))
# Configure the import to come, destination
db_writer = DBWriter(self.session)
import_where = ImportWhere(
db_writer, self.vault,
self.temp_for_jobs.base_dir_for(self.job_id))
# Configure the import to come, directives
import_how = ImportHow(self.prj_id, self.req.update_mode, col_mapping,
self.req.skip_existing_objects, loaded_files)
import_how.taxo_mapping = self.req.taxo_mappings
import_how.found_taxa = taxo_found
import_how.found_users = found_users
if self.req.skip_loaded_files:
import_how.compute_skipped(source_bundle, logger)
if self.req.skip_existing_objects:
# If we must skip existing objects then do an inventory of what's in already
with CodeTimer("run: Existing images for %d: " % self.prj_id,
logger):
import_how.objects_and_images_to_skip = Image.fetch_existing_images(
self.session, self.prj_id)
import_how.do_thumbnail_above(int(self.config['THUMBSIZELIMIT']))
# Do the bulk job of import
rowcount_from_validate = nb_rows
row_count = source_bundle.do_import(import_where, import_how,
rowcount_from_validate,
self.report_progress)
# Update loaded files in DB, removing duplicates
self.prj.fileloaded = "\n".join(set(import_how.loaded_files))
self.session.commit()
# Recompute stats
ProjectBO.do_after_load(self.session, self.prj_id)
self.session.commit()
msg = "Total of %d rows loaded" % row_count
logger.info(msg)
self.set_job_result(errors=[], infos={"rowcount": row_count})
def do_import(self):
"""
Do the real job, i.e. copy files while creating records.
"""
errors = []
self.manage_uploaded()
self.unzip_if_needed()
# Use a Bundle
source_bundle = InBundle(
self.source_dir_or_zip,
Path(self.temp_for_task.data_dir_for(self.task_id)))
# Clean it, in case the ZIP contains a CSV
source_bundle.remove_all_tsvs()
images = source_bundle.list_image_files()
# Configure the import to come, destination
db_writer = DBWriter(self.session)
import_where = ImportWhere(
db_writer, self.vault,
self.temp_for_task.base_dir_for(self.task_id))
# Configure the import to come, directives
import_how = ImportHow(prj_id=self.prj_id,
update_mode="",
custom_mapping=ProjectMapping(),
skip_object_duplicates=False,
loaded_files=[])
import_how.do_thumbnail_above(int(self.config['THUMBSIZELIMIT']))
# Generate TSV
req_values = self.req.values
if req_values.get(SimpleImportFields.userlb, ""):
import_how.found_users["user"] = {
"id": req_values.get(SimpleImportFields.userlb)
}
req_values[SimpleImportFields.userlb] = "user"
if req_values.get(SimpleImportFields.status, ""):
req_values[SimpleImportFields.status] = classif_qual.get(
req_values[SimpleImportFields.status], "")
self.make_tsv(source_bundle, images)
# Import
nb_image_files = len(images)
nb_images = source_bundle.do_import(import_where, import_how,
nb_image_files,
self.report_progress)
self.session.commit()
# Recompute stats and so on
ProjectBO.do_after_load(self.session, self.prj_id)
self.session.commit()
ret = SimpleImportRsp(errors=errors, nb_images=nb_images)
return ret
def do_run(self) -> None:
# OK
logger.info("Starting subset of '%s'", self.prj.title)
self.update_progress(5, "Determining objects to clone")
self._find_what_to_clone()
logger.info("Matched %s objects", len(self.to_clone))
if len(self.to_clone) == 0:
errors = ["No object found to clone into subset."]
self.set_job_result(errors=errors, infos={"infos": ""})
return
self._do_clone()
self.session.commit()
# Recompute stats and so on
ProjectBO.do_after_load(self.session, self.dest_prj.projid)
self.session.commit()
self.set_job_result(errors=[], infos={"rowcount": len(self.to_clone)})