def test_generate_fixity(self):
test_mock = mock.MagicMock()
result = "916f0027a575074ce72a331777c3478d6513f786a591bd892da1a577bf2335f9"
with mock.patch("builtins.open", test_mock):
manager = test_mock.return_value.__enter__.return_value
reads = ['test data', '']
manager.read.side_effect = lambda x: reads.pop(0).encode()
with open("test.rar") as f:
self.assertEquals(result, generate_fixity("testPath"),
"The actual hash differs from the expected")
def create_registry_entry(correlation_id):
"""
Creates an entry in the local registry when transfer
process is completed
:param correlation_id: String of the correlation_id used in IngestAction
"""
try:
ingest = IngestAction.objects.get(
correlation_id=correlation_id
)
except IngestAction.DoesNotExist as err:
raise err
replicating_nodes = ingest.sendfileaction_set.filter(
chosen_to_transfer=True,
step=COMPLETE,
state=SUCCESS
)
if replicating_nodes.count() > 0:
local_bag_path = os.path.join(DPN_INGEST_DIR_OUT, "%s.%s" % (
ingest.object_id, DPN_BAGS_FILE_EXT))
fixity_value = generate_fixity(local_bag_path)
now = datetime.now()
# attributes to update in registry entry
attributes = dict(
first_node_name=DPN_NODE_NAME,
version_number=1,
fixity_algorithm=DPN_FIXITY_CHOICES[0],
fixity_value=fixity_value,
last_fixity_date=now,
creation_date=now,
last_modified_date=now,
bag_size=os.path.getsize(local_bag_path)
)
try:
registry_entry = RegistryEntry.objects.get(
dpn_object_id=ingest.object_id)
# in case entry already exists, update its attributes
for attr, value in attributes.iteritems():
# this is to prevent modifying creation_date or last_fixity_date
# just update last_modified_date
if attr not in ['creation_date', 'last_fixity_date']:
setattr(registry_entry, attr, value)
# set flag created to false
created = False
except RegistryEntry.DoesNotExist as err:
attributes['dpn_object_id'] = ingest.object_id
registry_entry = RegistryEntry(**attributes)
created = True
# validate and save
registry_entry.full_clean()
registry_entry.save()
# now save replication nodes and own node
for node in [a.node for a in replicating_nodes] + [DPN_NODE_NAME]:
node, created = Node.objects.get_or_create(name=node)
registry_entry.replicating_nodes.add(node)
_status = "created" if created else "updated"
logger.info(
"Registry entry successfully %s for transaction with correlation_id: %s" %
(_status, correlation_id))
return registry_entry
else:
logger.info(
"Registry entry not created. The bag was not transferred by any node.")
return None
def verify_fixity_and_reply(req):
"""
Generates fixity value for local bag and compare it
with fixity value of the transferred bag. Sends a Replication
Verification Reply with ack or nak or retry according to
the fixities comparisons
:param req: ReplicationTransferReply already validated
"""
correlation_id = req.headers['correlation_id']
headers = {
'correlation_id': correlation_id,
'sequence': 5,
'date': dpn_strftime(datetime.now())
}
try:
action = SendFileAction.objects.get(
ingest__correlation_id=correlation_id,
node=req.headers['from'],
chosen_to_transfer=True
)
except SendFileAction.DoesNotExist as err:
logger.error(
"SendFileAction not found for correlation_id: %s. Exc msg: %s" % (
correlation_id, err))
raise DPNWorkflowError(err)
local_bag_path = os.path.join(settings.DPN_INGEST_DIR_OUT,
os.path.basename(action.location))
local_fixity = generate_fixity(local_bag_path)
if local_fixity == req.body['fixity_value']:
message_att = 'ack'
# save SendFileAction as complete and success
action.step = COMPLETE
action.state = SUCCESS
action.save()
print("Fixity value is correct. Correlation_id: %s" % correlation_id)
print("Creating registry entry...")
else:
message_att = 'nak'
# saving action status
action.step = VERIFY
action.state = FAILED
action.note = "Wrong fixity value of transferred bag. Sending nak verification reply"
action.save()
# TODO: under which condition should we retry the transfer?
# retry = {
# 'message_att': 'retry',
# }
body = dict(message_att=message_att)
rsp = ReplicationVerificationReply(headers, body)
rsp.send(req.headers['reply_key'])
# make sure to copy the bag to the settings.DPN_REPLICATION_ROOT folder
# to have it accesible in case of recovery request
ingested_bag = os.path.join(settings.DPN_REPLICATION_ROOT,
os.path.basename(action.location))
if not os.path.isfile(ingested_bag):
shutil.copy2(local_bag_path, settings.DPN_REPLICATION_ROOT)
# This task will create or update a registry entry
# for a given correlation_id. It is also linked with
# the sending of a item creation message
create_registry_entry.apply_async(
(correlation_id, ),
link=broadcast_item_creation.subtask()
)