def get_record_orig_or_mem(self, recid): # We upload patches at the end, so we're not picking up our own patches here # XXX Could put a lock around this but ugh. record = get_record_orig(recid) record_hash = hash(record) sorted_fullpatches = get_fullpatches(recid) for fullpatch in sorted_fullpatches: # FIXME: Check record hash? Or check in `get_fullpatches`? see conftest2.py:408 # assert fullpatch['record_hash'] == record_hash self.patches_we_applied_add(fullpatch) jsonpatch.apply_patch(record, fullpatch['patch'], in_place=True) return record
def handle_worker_completion(task_id): """Commit patches at the completion of a single worker. If `rule.confirm_hash_on_commit` is enabled then only records whose hash has not changed in the meantime are committed. ..note:: currently `CheckerRuleExecution.should_commit` depends on `dry_run`. :type task_id: str :param task_id: task_id as returned by `run_test` """ from invenio_records.api import get_record as get_record_orig with elioterize("finalize worker", worker_id=task_id): worker = RedisWorker(task_id) should_commit = worker.master.get_execution().should_commit Message.log(message_type='commit decision', commit=should_commit) recids_we_comitted_changes_to = intbitset() if should_commit: for recid, patches in worker.all_patches.items(): # recid: record ID # patches: patches for this record ID record = get_record_orig(recid) first_patch = True for patch in patches: # The record hash is bound to change once we've applied the # first hash, not to mention there is no reason to check # twice. if first_patch: first_patch = False if worker.master.rule.confirm_hash_on_commit: if hash(record) != patch['hash']: Message.log(message_type='skipping record', recid=record['id'], worker_id=task_id) break # No commits for this record, kthx. recids_we_comitted_changes_to += recid jsonpatch.apply_patch(record, patch, in_place=True) record.commit() Message.log(message_type='committing complete', patches_count=len(recids_we_comitted_changes_to)) worker.master.rule.mark_recids_as_checked( recids_we_comitted_changes_to) db.session.commit() worker.status = StatusWorker.committed
def get_record_orig_or_mem(self, recid): """Returns record from database, after applying existing patches. ..note:: We upload patches at the end, so we're not picking up our own patches here """ # XXX Should we put a lock around this? from invenio_records.api import get_record as get_record_orig record = get_record_orig(recid) sorted_fullpatches = get_sorted_fullpatches(recid, self.uuid) for fullpatch in sorted_fullpatches: jsonpatch.apply_patch(record, fullpatch['patch'], in_place=True) return record
def handle_worker_completion(task_id): """Commit patches at the completion of a single worker. If `rule.confirm_hash_on_commit` is enabled then only records whose hash has not changed in the meantime are committed. ..note:: currently `CheckerRuleExecution.should_commit` depends on `dry_run`. :type task_id: str :param task_id: task_id as returned by `run_test` """ from invenio_records.api import get_record as get_record_orig with elioterize("finalize worker", worker_id=task_id): worker = RedisWorker(task_id) should_commit = worker.master.get_execution().should_commit Message.log(message_type='commit decision', commit=should_commit) recids_we_comitted_changes_to = intbitset() if should_commit: for recid, patches in worker.all_patches.items(): # recid: record ID # patches: patches for this record ID record = get_record_orig(recid) first_patch = True for patch in patches: # The record hash is bound to change once we've applied the # first hash, not to mention there is no reason to check # twice. if first_patch: first_patch = False if worker.master.rule.confirm_hash_on_commit: if hash(record) != patch['hash']: Message.log(message_type='skipping record', recid=record['id'], worker_id=task_id) break # No commits for this record, kthx. recids_we_comitted_changes_to += recid jsonpatch.apply_patch(record, patch, in_place=True) record.commit() Message.log(message_type='committing complete', patches_count=len(recids_we_comitted_changes_to)) worker.master.rule.mark_recids_as_checked(recids_we_comitted_changes_to) db.session.commit() worker.status = StatusWorker.committed
def get_record_orig_or_mem(self, recid): """Returns record from database, after applying existing patches. ..note:: We upload patches at the end, so we're not picking up our own patches here """ # XXX Should we put a lock around this? from invenio_records.api import get_record as get_record_orig record = get_record_orig(recid) sorted_fullpatches = get_sorted_fullpatches(recid, self.uuid) for fullpatch in sorted_fullpatches: jsonpatch.apply_patch(record, fullpatch["patch"], in_place=True) return record
def handle_results(task_ids): """Commit patches. :type task_ids: list of str :param task_ids: values returned by `run_test` instances """ with with_eliot(action_type='handle results', worker_id=task_ids[0]): for task_id in task_ids: redis_worker = RedisWorker(task_id) for recid, patches in redis_worker.all_patches.items(): record = get_record_orig(recid) for patch in patches: jsonpatch.apply_patch(record, patch, in_place=True) record.commit() redis_worker.master.rule.mark_recids_as_checked(redis_worker.bundle_requested_recids) redis_worker.status = StatusWorker.committed
def handle_results(task_ids): """Commit patches. :type task_ids: list of str :param task_ids: values returned by `run_test` instances """ with with_eliot(action_type='handle results', worker_id=task_ids[0]): for task_id in task_ids: redis_worker = RedisWorker(task_id) for recid, patches in redis_worker.all_patches.items(): record = get_record_orig(recid) for patch in patches: jsonpatch.apply_patch(record, patch, in_place=True) record.commit() redis_worker.master.rule.mark_recids_as_checked( redis_worker.bundle_requested_recids) redis_worker.status = StatusWorker.committed