def setUp(self):
self.conn = Connection("memory://")
self.csmr = DPNConsumer(
self.conn,
DPN_EXCHANGE,
DPN_BROADCAST_QUEUE,
DPN_BROADCAST_KEY,
DPN_LOCAL_QUEUE,
DPN_LOCAL_KEY,
broadcast_rtr=MockRouter(),
local_rtr=MockRouter()
)
self.bad_msg = Message(Mock(), '{"message_name": "test"}')
self.expired_msg = Message(Mock(), '{"message_name": "test"}',
headers={
"ttl": dpn_strftime(datetime.now() - timedelta(days=3)),
"from": "test"
},
)
self.good_msg = Message(Mock(), '{"message_name": "test"}',
headers={
"ttl": dpn_strftime(datetime.now() + timedelta(days=3)),
"from": "test"
},
)
def handle(self, *args, **options):
headers = {
'correlation_id': uuid(),
'sequence': 0,
'date': dpn_strftime(datetime.now()),
}
body = {
"message_name": "registry-item-create",
"dpn_object_id": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
"local_id": "APTRUST-282dcbdd-c16b-42f1-8c21-0dd7875fb94e",
"first_node_name": "aptrust",
"replicating_node_names": ["hathi", "chron", "sdr"],
"version_number": 1,
"previous_version_object_id": "null",
"forward_version_object_id": "null",
"first_version_object_id": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
"fixity_algorithm": "sha256",
"fixity_value": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
"lastfixity_date": "2013-01-18T09:49:28-0800",
"creation_date": "2013-01-05T09:49:28-0800",
"last_modified_date": "2013-01-05T09:49:28-0800",
"bag_size": 65536,
"brightening_object_id": ["a02de3cd-a74b-4cc6-adec-16f1dc65f726",
"C92de3cd-a789-4cc6-adec-16a40c65f726", ],
"rights_object_id": ["0df688d4-8dfb-4768-bee9-639558f40488", ],
"object_type": "data",
}
msg = RegistryItemCreate(headers, body)
msg.send(DPN_BROADCAST_KEY)
def _get_headers(correlation_id, sequence):
return {
'correlation_id': correlation_id,
'date': dpn_strftime(datetime.now()),
'ttl': str_expire_on(datetime.now()),
'sequence': sequence
}
def make_headers():
header = {
'from': 'testfrom',
'reply_key': 'testkey',
'correlation_id': 'testid',
'sequence': 10,
'date': dpn_strftime(datetime.now()),
'ttl': str_expire_on(datetime.now(), 566),
}
return header
def respond_to_replication_query(init_request):
"""
Verifies if current node is available and has enough storage
to replicate bags and sends a ReplicationAvailableReply.
:param init_request: ReplicationInitQuery already validated
"""
# Prep Reply
headers = {
'correlation_id': init_request.headers['correlation_id'],
'date': dpn_strftime(datetime.now()),
'ttl': str_expire_on(datetime.now()),
'sequence': 1
}
body = {
'message_att': 'nak'
}
sequence_info = store_sequence(
headers['correlation_id'],
init_request.headers['from'],
headers['sequence']
)
validate_sequence(sequence_info)
bag_size = init_request.body['replication_size']
avail_storage = available_storage(DPN_REPLICATION_ROOT)
supported_protocols = [val for val in init_request.body['protocol']
if val in DPN_XFER_OPTIONS]
if supported_protocols and \
bag_size < avail_storage and \
bag_size < DPN_MAX_SIZE:
try:
protocol = protocol_str2db(supported_protocols[0])
receive_available_workflow(
node=init_request.headers["from"],
protocol=protocol,
id=init_request.headers["correlation_id"]
)
body = {
'message_att': 'ack',
'protocol': DPN_DEFAULT_XFER_PROTOCOL
}
except ValidationError as err:
logger.info('ValidationError: %s' % err)
pass # Record not created nak sent
except DPNWorkflowError as err:
logger.info('DPN Workflow Error: %s' % err)
pass # Record not created, nak sent
rsp = ReplicationAvailableReply(headers, body)
rsp.send(init_request.headers['reply_key'])
def handle(self, *args, **options):
start_datetime = self.validate_date(options['startdate'])
end_datetime = self.validate_date(options['enddate'])
if start_datetime > end_datetime:
raise CommandError("Start date must be prior to End Date")
headers = {
'correlation_id': str(uuid4()),
'sequence': 0
}
body = {
'date_range': [dpn_strftime(start_datetime), dpn_strftime(end_datetime)]
}
reg_sync = RegistryDateRangeSync(headers, body)
reg_sync.send(DPN_BROADCAST_KEY)
delay = DPN_MSG_TTL.get("registry-daterange-sync-request", DPN_TTL)
solve_registry_conflicts.apply_async(countdown=delay)
def as_dpn_dict(self):
"""
Returns a dictionary formatted fields filtered as needed
for DPN message bodies.
:param filters: Any non-argument callable.
:return: string of the filtered cleaned_data as json.
"""
data = map_to_json(self.field_map, self.cleaned_data.copy())
for k, v in data.items():
if type(v) is datetime.datetime:
data[k] = dpn_strftime(v)
if v is None:
data[k] = "null" # wierd requirement in dpn to use a string.
return data
def handle(self, *args, **options):
msg = ReplicationInitQuery()
headers = {
'correlation_id': uuid(),
'sequence': 0,
'date': dpn_strftime(datetime.now())
}
msg.set_headers(**headers)
body = {
'message_name': 'replication-init-query',
'replication_size': 4502,
'protocol': ['https', 'rsync'],
'dpn_object_id': uuid()
}
msg.set_body(**body)
msg.send(DPN_BROADCAST_KEY)
def initiate_ingest(dpn_object_id, size):
"""
Initiates an ingest operation by minting the correlation ID
:param dpn_object_id: UUID of the DPN object to send to the federation (extracted from bag filename)
:param size: Integer of the bag size
:return: Correlation ID to be used by choose_and_send_location linked task.
"""
# NOTE: Before sending the request, when this is real, it should...
# 1. Stage or confirm presence of bag in the staging area.
# 2. Validate the bag before sending.
action = IngestAction(
correlation_id=str(uuid4()),
object_id=dpn_object_id,
state=STARTED
)
headers = {
"correlation_id": action.correlation_id,
"date": dpn_strftime(datetime.now()),
"ttl": str_expire_on(datetime.now()),
"sequence": 0
}
body = {
"replication_size": size,
"protocol": settings.DPN_XFER_OPTIONS,
"dpn_object_id": dpn_object_id
}
sequence_info = store_sequence(headers['correlation_id'],
settings.DPN_NODE_NAME,
headers['sequence'])
validate_sequence(sequence_info)
try:
msg = ReplicationInitQuery(headers, body)
msg.send(settings.DPN_BROADCAST_KEY)
action.state = SUCCESS
except Exception as err:
action.state = FAILED
action.note = "%s" % err
logger.error(err)
action.save()
return action.correlation_id
def _is_alive(self, msg, current_time):
"""
Check if the message has date and ttl set and returns if still alive
:param msg: kombu.transport.base.Message instance.
:return: Boolean
"""
try:
ttl = dpn_strptime(msg.headers['ttl'])
now = dpn_strptime(dpn_strftime(current_time))
return ttl > now
except KeyError:
msg.ack()
raise DPNMessageError(
"Invalid message received with no 'date' or 'ttl' set!")
def as_dpn_dict(self):
data = map_to_json(self.field_map, self.clean())
# Convert None fields to 'null'
for fieldname in self.default_null:
if data[fieldname] == None or data[fieldname] == "":
data[fieldname] = 'null'
# Convert Datetime values to DPN string format.
for k, v in data.items():
if type(v) is datetime.datetime:
data[k] = dpn_strftime(v)
if isinstance(v, models.Model):
data[k] = "%s" % v
if isinstance(v, QuerySet):
data[k] = ["%s" % value for value in v]
# Convert object_type to the output value.
types = dict([(key, value.lower()) for (key, value) in TYPE_CHOICES])
data['object_type'] = types[data['object_type']]
return data
def test_dpn_strftime(self):
self.failUnlessEqual(self.timestring, dpn_strftime(self.datetime))
def validate_date(self, datestring):
try:
return dpn_strptime(datestring)
except ValueError:
raise ValueError("Incorrect date format, should be '%s' (i.e. %s)" % DPN_DATE_FORMAT,
dpn_strftime(datetime.utcnow()))
def respond_to_recovery_query(init_request):
"""
Verifies if current node is a first node and the bag is in
the node and sends a RecoveryAvailableReply.
:param init_request: RecoveryInitQuery already validated
"""
correlation_id = init_request.headers['correlation_id']
node_from = init_request.headers['from']
dpn_object_id = init_request.body['dpn_object_id']
reply_key = init_request.headers['reply_key']
sequence = 1
note = None
# Prep Reply
headers = _get_headers(correlation_id, sequence)
body = {
'message_att': 'nak'
}
_validate_sequence(correlation_id, node_from, sequence)
# Verifies that the sender node is either a First or a Replication Node
try:
node = Node.objects.get(name=node_from)
entry = RegistryEntry.objects.get(dpn_object_id=dpn_object_id)
if node.name == entry.first_node_name or node in entry.replicating_nodes.all():
supported_protocols = [val for val in init_request.body['protocol']
if val in settings.DPN_XFER_OPTIONS]
if supported_protocols:
# Verifies that the content is available
basefile = "{0}.{1}".format(dpn_object_id,
settings.DPN_BAGS_FILE_EXT)
local_bagfile = os.path.join(settings.DPN_REPLICATION_ROOT, basefile)
if os.path.isfile(local_bagfile):
body = {
"available_at": dpn_strftime(datetime.now()),
"message_att": "ack",
"protocol": supported_protocols[0],
"cost": 0
}
else:
note = "The content is not available"
else:
note = "The protocol is not supported"
else:
note = "Current node is not listed either as first node or replicating node."
except Node.DoesNotExist:
note = "The node does not exists"
# If working with just one server, the action has been stored in the database
action, _ = Workflow.objects.get_or_create(
correlation_id=correlation_id,
dpn_object_id=dpn_object_id,
node=node_from,
action=RECOVERY
)
action.step = AVAILABLE_REPLY
action.reply_key = reply_key
action.note = note
action.state = SUCCESS if body["message_att"] == "ack" else FAILED
# Sends the reply
rsp = RecoveryAvailableReply(headers, body)
rsp.send(reply_key)
# Save the action in DB
action.save()
def verify_fixity_and_reply(req):
"""
Generates fixity value for local bag and compare it
with fixity value of the transferred bag. Sends a Replication
Verification Reply with ack or nak or retry according to
the fixities comparisons
:param req: ReplicationTransferReply already validated
"""
correlation_id = req.headers['correlation_id']
headers = {
'correlation_id': correlation_id,
'sequence': 5,
'date': dpn_strftime(datetime.now())
}
try:
action = SendFileAction.objects.get(
ingest__correlation_id=correlation_id,
node=req.headers['from'],
chosen_to_transfer=True
)
except SendFileAction.DoesNotExist as err:
logger.error(
"SendFileAction not found for correlation_id: %s. Exc msg: %s" % (
correlation_id, err))
raise DPNWorkflowError(err)
local_bag_path = os.path.join(settings.DPN_INGEST_DIR_OUT,
os.path.basename(action.location))
local_fixity = generate_fixity(local_bag_path)
if local_fixity == req.body['fixity_value']:
message_att = 'ack'
# save SendFileAction as complete and success
action.step = COMPLETE
action.state = SUCCESS
action.save()
print("Fixity value is correct. Correlation_id: %s" % correlation_id)
print("Creating registry entry...")
else:
message_att = 'nak'
# saving action status
action.step = VERIFY
action.state = FAILED
action.note = "Wrong fixity value of transferred bag. Sending nak verification reply"
action.save()
# TODO: under which condition should we retry the transfer?
# retry = {
# 'message_att': 'retry',
# }
body = dict(message_att=message_att)
rsp = ReplicationVerificationReply(headers, body)
rsp.send(req.headers['reply_key'])
# make sure to copy the bag to the settings.DPN_REPLICATION_ROOT folder
# to have it accesible in case of recovery request
ingested_bag = os.path.join(settings.DPN_REPLICATION_ROOT,
os.path.basename(action.location))
if not os.path.isfile(ingested_bag):
shutil.copy2(local_bag_path, settings.DPN_REPLICATION_ROOT)
# This task will create or update a registry entry
# for a given correlation_id. It is also linked with
# the sending of a item creation message
create_registry_entry.apply_async(
(correlation_id, ),
link=broadcast_item_creation.subtask()
)
