def setUp(self): self.conn = Connection("memory://") self.csmr = DPNConsumer( self.conn, DPN_EXCHANGE, DPN_BROADCAST_QUEUE, DPN_BROADCAST_KEY, DPN_LOCAL_QUEUE, DPN_LOCAL_KEY, broadcast_rtr=MockRouter(), local_rtr=MockRouter() ) self.bad_msg = Message(Mock(), '{"message_name": "test"}') self.expired_msg = Message(Mock(), '{"message_name": "test"}', headers={ "ttl": dpn_strftime(datetime.now() - timedelta(days=3)), "from": "test" }, ) self.good_msg = Message(Mock(), '{"message_name": "test"}', headers={ "ttl": dpn_strftime(datetime.now() + timedelta(days=3)), "from": "test" }, )
def handle(self, *args, **options): headers = { 'correlation_id': uuid(), 'sequence': 0, 'date': dpn_strftime(datetime.now()), } body = { "message_name": "registry-item-create", "dpn_object_id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", "local_id": "APTRUST-282dcbdd-c16b-42f1-8c21-0dd7875fb94e", "first_node_name": "aptrust", "replicating_node_names": ["hathi", "chron", "sdr"], "version_number": 1, "previous_version_object_id": "null", "forward_version_object_id": "null", "first_version_object_id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", "fixity_algorithm": "sha256", "fixity_value": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824", "lastfixity_date": "2013-01-18T09:49:28-0800", "creation_date": "2013-01-05T09:49:28-0800", "last_modified_date": "2013-01-05T09:49:28-0800", "bag_size": 65536, "brightening_object_id": ["a02de3cd-a74b-4cc6-adec-16f1dc65f726", "C92de3cd-a789-4cc6-adec-16a40c65f726", ], "rights_object_id": ["0df688d4-8dfb-4768-bee9-639558f40488", ], "object_type": "data", } msg = RegistryItemCreate(headers, body) msg.send(DPN_BROADCAST_KEY)
def _get_headers(correlation_id, sequence): return { 'correlation_id': correlation_id, 'date': dpn_strftime(datetime.now()), 'ttl': str_expire_on(datetime.now()), 'sequence': sequence }
def make_headers(): header = { 'from': 'testfrom', 'reply_key': 'testkey', 'correlation_id': 'testid', 'sequence': 10, 'date': dpn_strftime(datetime.now()), 'ttl': str_expire_on(datetime.now(), 566), } return header
def respond_to_replication_query(init_request): """ Verifies if current node is available and has enough storage to replicate bags and sends a ReplicationAvailableReply. :param init_request: ReplicationInitQuery already validated """ # Prep Reply headers = { 'correlation_id': init_request.headers['correlation_id'], 'date': dpn_strftime(datetime.now()), 'ttl': str_expire_on(datetime.now()), 'sequence': 1 } body = { 'message_att': 'nak' } sequence_info = store_sequence( headers['correlation_id'], init_request.headers['from'], headers['sequence'] ) validate_sequence(sequence_info) bag_size = init_request.body['replication_size'] avail_storage = available_storage(DPN_REPLICATION_ROOT) supported_protocols = [val for val in init_request.body['protocol'] if val in DPN_XFER_OPTIONS] if supported_protocols and \ bag_size < avail_storage and \ bag_size < DPN_MAX_SIZE: try: protocol = protocol_str2db(supported_protocols[0]) receive_available_workflow( node=init_request.headers["from"], protocol=protocol, id=init_request.headers["correlation_id"] ) body = { 'message_att': 'ack', 'protocol': DPN_DEFAULT_XFER_PROTOCOL } except ValidationError as err: logger.info('ValidationError: %s' % err) pass # Record not created nak sent except DPNWorkflowError as err: logger.info('DPN Workflow Error: %s' % err) pass # Record not created, nak sent rsp = ReplicationAvailableReply(headers, body) rsp.send(init_request.headers['reply_key'])
def handle(self, *args, **options): start_datetime = self.validate_date(options['startdate']) end_datetime = self.validate_date(options['enddate']) if start_datetime > end_datetime: raise CommandError("Start date must be prior to End Date") headers = { 'correlation_id': str(uuid4()), 'sequence': 0 } body = { 'date_range': [dpn_strftime(start_datetime), dpn_strftime(end_datetime)] } reg_sync = RegistryDateRangeSync(headers, body) reg_sync.send(DPN_BROADCAST_KEY) delay = DPN_MSG_TTL.get("registry-daterange-sync-request", DPN_TTL) solve_registry_conflicts.apply_async(countdown=delay)
def as_dpn_dict(self): """ Returns a dictionary formatted fields filtered as needed for DPN message bodies. :param filters: Any non-argument callable. :return: string of the filtered cleaned_data as json. """ data = map_to_json(self.field_map, self.cleaned_data.copy()) for k, v in data.items(): if type(v) is datetime.datetime: data[k] = dpn_strftime(v) if v is None: data[k] = "null" # wierd requirement in dpn to use a string. return data
def handle(self, *args, **options): msg = ReplicationInitQuery() headers = { 'correlation_id': uuid(), 'sequence': 0, 'date': dpn_strftime(datetime.now()) } msg.set_headers(**headers) body = { 'message_name': 'replication-init-query', 'replication_size': 4502, 'protocol': ['https', 'rsync'], 'dpn_object_id': uuid() } msg.set_body(**body) msg.send(DPN_BROADCAST_KEY)
def initiate_ingest(dpn_object_id, size): """ Initiates an ingest operation by minting the correlation ID :param dpn_object_id: UUID of the DPN object to send to the federation (extracted from bag filename) :param size: Integer of the bag size :return: Correlation ID to be used by choose_and_send_location linked task. """ # NOTE: Before sending the request, when this is real, it should... # 1. Stage or confirm presence of bag in the staging area. # 2. Validate the bag before sending. action = IngestAction( correlation_id=str(uuid4()), object_id=dpn_object_id, state=STARTED ) headers = { "correlation_id": action.correlation_id, "date": dpn_strftime(datetime.now()), "ttl": str_expire_on(datetime.now()), "sequence": 0 } body = { "replication_size": size, "protocol": settings.DPN_XFER_OPTIONS, "dpn_object_id": dpn_object_id } sequence_info = store_sequence(headers['correlation_id'], settings.DPN_NODE_NAME, headers['sequence']) validate_sequence(sequence_info) try: msg = ReplicationInitQuery(headers, body) msg.send(settings.DPN_BROADCAST_KEY) action.state = SUCCESS except Exception as err: action.state = FAILED action.note = "%s" % err logger.error(err) action.save() return action.correlation_id
def _is_alive(self, msg, current_time): """ Check if the message has date and ttl set and returns if still alive :param msg: kombu.transport.base.Message instance. :return: Boolean """ try: ttl = dpn_strptime(msg.headers['ttl']) now = dpn_strptime(dpn_strftime(current_time)) return ttl > now except KeyError: msg.ack() raise DPNMessageError( "Invalid message received with no 'date' or 'ttl' set!")
def as_dpn_dict(self): data = map_to_json(self.field_map, self.clean()) # Convert None fields to 'null' for fieldname in self.default_null: if data[fieldname] == None or data[fieldname] == "": data[fieldname] = 'null' # Convert Datetime values to DPN string format. for k, v in data.items(): if type(v) is datetime.datetime: data[k] = dpn_strftime(v) if isinstance(v, models.Model): data[k] = "%s" % v if isinstance(v, QuerySet): data[k] = ["%s" % value for value in v] # Convert object_type to the output value. types = dict([(key, value.lower()) for (key, value) in TYPE_CHOICES]) data['object_type'] = types[data['object_type']] return data
def test_dpn_strftime(self): self.failUnlessEqual(self.timestring, dpn_strftime(self.datetime))
def validate_date(self, datestring): try: return dpn_strptime(datestring) except ValueError: raise ValueError("Incorrect date format, should be '%s' (i.e. %s)" % DPN_DATE_FORMAT, dpn_strftime(datetime.utcnow()))
def respond_to_recovery_query(init_request): """ Verifies if current node is a first node and the bag is in the node and sends a RecoveryAvailableReply. :param init_request: RecoveryInitQuery already validated """ correlation_id = init_request.headers['correlation_id'] node_from = init_request.headers['from'] dpn_object_id = init_request.body['dpn_object_id'] reply_key = init_request.headers['reply_key'] sequence = 1 note = None # Prep Reply headers = _get_headers(correlation_id, sequence) body = { 'message_att': 'nak' } _validate_sequence(correlation_id, node_from, sequence) # Verifies that the sender node is either a First or a Replication Node try: node = Node.objects.get(name=node_from) entry = RegistryEntry.objects.get(dpn_object_id=dpn_object_id) if node.name == entry.first_node_name or node in entry.replicating_nodes.all(): supported_protocols = [val for val in init_request.body['protocol'] if val in settings.DPN_XFER_OPTIONS] if supported_protocols: # Verifies that the content is available basefile = "{0}.{1}".format(dpn_object_id, settings.DPN_BAGS_FILE_EXT) local_bagfile = os.path.join(settings.DPN_REPLICATION_ROOT, basefile) if os.path.isfile(local_bagfile): body = { "available_at": dpn_strftime(datetime.now()), "message_att": "ack", "protocol": supported_protocols[0], "cost": 0 } else: note = "The content is not available" else: note = "The protocol is not supported" else: note = "Current node is not listed either as first node or replicating node." except Node.DoesNotExist: note = "The node does not exists" # If working with just one server, the action has been stored in the database action, _ = Workflow.objects.get_or_create( correlation_id=correlation_id, dpn_object_id=dpn_object_id, node=node_from, action=RECOVERY ) action.step = AVAILABLE_REPLY action.reply_key = reply_key action.note = note action.state = SUCCESS if body["message_att"] == "ack" else FAILED # Sends the reply rsp = RecoveryAvailableReply(headers, body) rsp.send(reply_key) # Save the action in DB action.save()
def verify_fixity_and_reply(req): """ Generates fixity value for local bag and compare it with fixity value of the transferred bag. Sends a Replication Verification Reply with ack or nak or retry according to the fixities comparisons :param req: ReplicationTransferReply already validated """ correlation_id = req.headers['correlation_id'] headers = { 'correlation_id': correlation_id, 'sequence': 5, 'date': dpn_strftime(datetime.now()) } try: action = SendFileAction.objects.get( ingest__correlation_id=correlation_id, node=req.headers['from'], chosen_to_transfer=True ) except SendFileAction.DoesNotExist as err: logger.error( "SendFileAction not found for correlation_id: %s. Exc msg: %s" % ( correlation_id, err)) raise DPNWorkflowError(err) local_bag_path = os.path.join(settings.DPN_INGEST_DIR_OUT, os.path.basename(action.location)) local_fixity = generate_fixity(local_bag_path) if local_fixity == req.body['fixity_value']: message_att = 'ack' # save SendFileAction as complete and success action.step = COMPLETE action.state = SUCCESS action.save() print("Fixity value is correct. Correlation_id: %s" % correlation_id) print("Creating registry entry...") else: message_att = 'nak' # saving action status action.step = VERIFY action.state = FAILED action.note = "Wrong fixity value of transferred bag. Sending nak verification reply" action.save() # TODO: under which condition should we retry the transfer? # retry = { # 'message_att': 'retry', # } body = dict(message_att=message_att) rsp = ReplicationVerificationReply(headers, body) rsp.send(req.headers['reply_key']) # make sure to copy the bag to the settings.DPN_REPLICATION_ROOT folder # to have it accesible in case of recovery request ingested_bag = os.path.join(settings.DPN_REPLICATION_ROOT, os.path.basename(action.location)) if not os.path.isfile(ingested_bag): shutil.copy2(local_bag_path, settings.DPN_REPLICATION_ROOT) # This task will create or update a registry entry # for a given correlation_id. It is also linked with # the sending of a item creation message create_registry_entry.apply_async( (correlation_id, ), link=broadcast_item_creation.subtask() )