Beispiel #1
0
 def test_update_new(self):
     """
     Simulate the case where synchronization brings a new storage object to be instantiated.
     """
     # Exercise
     update_resource(self.storage_json, self.metadata_before, self.storage_digest)
     # Verify
     self.assertEquals(1, StorageObject.objects.filter(identifier=self.storage_id).count())
     storage_object = StorageObject.objects.get(identifier=self.storage_id)
     self.assertEquals(self.metadata_before, storage_object.metadata)
Beispiel #2
0
 def test_master_copy(self):
     """
     Verify that reusable entities such as persons have copy status MASTER
     after synchronization.
     """
     # exercise
     update_resource(self.storage_json, self.metadata_before, None, copy_status=MASTER)
     # verify
     resource = resourceInfoType_model.objects.get(storage_object__identifier=self.storage_id)
     persons = resource.contactPerson.all()
     self.assertEquals(1, len(persons))
     contact_person = persons[0]
     self.assertEquals(MASTER, contact_person.copy_status)
Beispiel #3
0
 def test_update_refuse_mastercopy(self):
     """
     Refuse to replace a master copy with a non-master copy during update 
     """
     # setup
     update_resource(self.storage_json, self.metadata_before, self.storage_digest, MASTER)
     self.assertEquals(MASTER, StorageObject.objects.get(identifier=self.storage_id).copy_status)
     # exercise
     try:
         update_resource(self.storage_json, self.metadata_modified, self.storage_digest, REMOTE)
         self.fail("Should have raised an exception")
     except IllegalAccessException:
         pass # Expected exception
Beispiel #4
0
 def test_update_existing(self):
     """
     Simulate update for already existing storage object
     """
     # helper
     def get_metadatacreationdate_for(storage_id):
         resource = resourceInfoType_model.objects.get(storage_object__identifier=self.storage_id)
         return resource.metadataInfo.metadataCreationDate
     
     # setup
     update_resource(self.storage_json, self.metadata_before, self.storage_digest)
     self.assertEquals(date(2005, 5, 12), get_metadatacreationdate_for(self.storage_id))
     self.assertEquals(REMOTE, StorageObject.objects.get(identifier=self.storage_id).copy_status)
     # exercise
     update_resource(self.storage_json, self.metadata_modified, self.storage_digest)
     self.assertEquals(date(2006, 12, 31), get_metadatacreationdate_for(self.storage_id))
Beispiel #5
0
    def handle(self, *args, **options):
        # Get the list of the servers to be querried
        core_nodes = CORE_NODES
        #print core_nodes

        
        for server in core_nodes.values():
            new_resources = []
            resources_to_update = []
            local_inventory = []

            # Login
            url = server['URL']
            user_name = server['USERNAME']
            password = server['PASSWORD']
            opener = login("{0}/login/".format(url), user_name, password)
            
            # Get the inventory list. 
            remote_inventory = get_inventory(opener, "{0}/sync/".format(url))
            remote_inventory_count = len(remote_inventory)
            sys.stdout.write("\nRemote node " + BOLD + url + RESET + " contains " + BOLD + str(remote_inventory_count) + " resources.\n" + RESET)
            
            # Get a list of uuid's and digests from the local inventory
            non_master_storage_objects = StorageObject.objects.exclude(copy_status=MASTER)
            for item in non_master_storage_objects:
                local_inventory.append({'id':str(item.identifier), 'digest':str(item.digest_checksum)})
            #print "\nLOCAL INVENTORY: \n" + str(local_inventory)
            local_inventory_count = len(local_inventory)
            sys.stdout.write("\nLocal node contains " + BOLD + str(local_inventory_count) + " resources.\n" + RESET)
            
            
            # Create an list of ids to speed-up matching
            local_inventory_indexed = []
            for item in local_inventory:
                local_inventory_indexed.append(item['id'])
            #print "\nINVENTORY LIST : \n" + str(local_inventory_indexed)
            
            # Create two lists:
            # 1. Containing items to be added - items that exist in the 
            # remote inventory and not in the local.
            # 2. Containing items to be updated - items that exist in both
            # inventories but the remote is different from the local
            for item in remote_inventory:
                item_id = item['id']
                if item_id not in local_inventory_indexed:
                    new_resources.append(item)
                else:
                    # Find the corresponding item in the local inventory
                    # and compare digests
                    for local_item in local_inventory:
                        if (item_id == local_item['id']) \
                          and not (item['digest'] == local_item['digest']):
                            resources_to_update.append(item)

            new_resources_count = len(new_resources)
            resources_to_update_count = len(resources_to_update)
            
            print new_resources
            
            sys.stdout.write("\n " + BOLD + str(new_resources_count) + \
              " new resources"  + RESET + " will be added to your repository.\n")
            sys.stdout.write("\n " + BOLD + str(resources_to_update_count) + \
              " resources"  + RESET + " will be updated in your repository.\n")
            
            sys.stdout.write("\n Importing and Indexing...\n")
            
            
            for resource in new_resources:
                # Get the json storage object and the actual metadata xml
                storage_json, resource_xml_string = get_full_metadata(opener, "{0}/sync/{1}/metadata/".format(url, resource['id']), resource['digest'])
                update_resource(storage_json, resource_xml_string)
            
            for resource in resources_to_update:
                # Get the json storage object and the actual metadata xml
                storage_json, resource_xml_string = get_full_metadata(opener, "{0}/sync/{1}/metadata/".format(url, resource['id']), resource['digest'])
                update_resource(storage_json, resource_xml_string)
Beispiel #6
0
 def sync_with_server(server, is_proxy, id_file=None):
     """
     Synchronizes this META-SHARE node with another META-SHARE node using
     the given server description.
     
     `nodes` is a dict of dicts with synchronization settings for the nodes
         to synchronize with
     `is_proxy` must be True if this node is a proxy for the given nodes;
         it must be False if the given nodes are not proxied by this node
     """
     
     new_resources = []
     resources_to_update = []
     local_inventory = []
     
     # Login
     url = server['URL']
     user_name = server['USERNAME']
     password = server['PASSWORD']
     opener = login("{0}/login/".format(url), user_name, password)
     
     # Get the inventory list. 
     remote_inventory = get_inventory(opener, "{0}/sync/".format(url))
     
     # handle existing resources
     remote_inventory_existing = remote_inventory['existing']
     remote_inventory_count = len(remote_inventory_existing)
     sys.stdout.write("\nRemote node " + BOLD + url + RESET + " contains " \
       + BOLD + str(remote_inventory_count) + " resources.\n" + RESET)
     
     # Get a list of uuid's and digests from the local inventory
     non_master_storage_objects = StorageObject.objects.exclude(copy_status=MASTER)
     for item in non_master_storage_objects:
         local_inventory.append({'id':str(item.identifier), 'digest':str(item.digest_checksum)})
     local_inventory_count = len(local_inventory)
     sys.stdout.write("\nLocal node contains " + BOLD + str(local_inventory_count) \
       + " resources.\n" + RESET)
     
     # Create an list of ids to speed-up matching
     local_inventory_indexed = []
     for item in local_inventory:
         local_inventory_indexed.append(item['id'])
     
     # Create two lists:
     # 1. Containing items to be added - items that exist in the 
     # remote inventory and not in the local.
     # 2. Containing items to be updated - items that exist in both
     # inventories but the remote is different from the local
     for item in remote_inventory_existing:
         item_id = item['id']
         if item_id not in local_inventory_indexed:
             new_resources.append(item)
         else:
             # Find the corresponding item in the local inventory
             # and compare digests
             for local_item in local_inventory:
                 if (item_id == local_item['id']) \
                   and not (item['digest'] == local_item['digest']):
                     resources_to_update.append(item)
     
     
     # Print informative messages to the user
     new_resources_count = len(new_resources)
     resources_to_update_count = len(resources_to_update)            
     if ((new_resources_count == 0) and (resources_to_update_count == 0)):
         sys.stdout.write("\nThere are no resources marked" +\
           " for updating!\n")
     else:
         # If there are resources to add or update
         sys.stdout.write("\n" + BOLD + \
           ("No" if new_resources_count == 0 \
           else str(new_resources_count)) + \
           " new resource" + ("" if new_resources_count == 1 else "s") \
           + RESET + " will be added to your repository.\n")
         sys.stdout.write("\n" + BOLD + \
           ("No" if resources_to_update_count == 0 \
           else str(resources_to_update_count)) + \
           " resource" + ("" if resources_to_update_count == 1 else "s") \
           + RESET + " will be updated in your repository.\n")
         sys.stdout.write("\nImporting and Indexing...\n")
     
         if is_proxy:
             _copy_status = PROXY
         else:
             _copy_status = REMOTE
         
         # Get the full xmls from remore inventory and update local inventory
         for resource in new_resources:
             # Get the json storage object and the actual metadata xml
             storage_json, resource_xml_string = \
               get_full_metadata(opener, "{0}/sync/{1}/metadata/" \
                     .format(url, resource['id']), resource['digest'])
             res_obj = update_resource(storage_json, resource_xml_string,
                             resource['digest'], _copy_status)
             if not id_file is None:
                 id_file.write("--->RESOURCE_ID:{0};STORAGE_IDENTIFIER:{1}\n"\
                     .format(res_obj.id, res_obj.storage_object.identifier))
     
         for resource in resources_to_update:
             # Get the json storage object and the actual metadata xml
             storage_json, resource_xml_string = \
               get_full_metadata(opener, "{0}/sync/{1}/metadata/" \
                     .format(url, resource['id']), resource['digest'])
             res_obj = update_resource(storage_json, resource_xml_string,
                             resource['digest'], _copy_status)
             if not id_file is None:
                 id_file.write("--->RESOURCE_ID:{0};STORAGE_IDENTIFIER:{1}\n"\
                     .format(res_obj.id, res_obj.storage_object.identifier))
                 if resource['digest'] != res_obj.storage_object.digest_checksum:
                     id_file.write("Different digests!\n")
     
     sys.stdout.write("\n\n")
     
     # handle removed resources
     remote_inventory_removed = remote_inventory['removed']
     remote_inventory_removed_count = len(remote_inventory_removed)
     sys.stdout.write("\nRemote node " + BOLD + url + RESET + " lists " \
       + BOLD + str(remote_inventory_removed_count) + " resources as removed.\n" + RESET)
     
     removed_count = 0
     for removed_id in remote_inventory_removed:
         if removed_id in local_inventory_indexed:
             # remove resource from this node;
             # if it is a PROXY copy, also create a corresponding removed
             # object, so that the removal is propagated to other
             # META-SHARE Managing Nodes (aka. inner nodes)
             sys.stdout.write("\nRemoving id {}...\n".format(removed_id))
             removed_count += 1
             _so_to_remove = StorageObject.objects.get(identifier=removed_id)
             if _so_to_remove.copy_status is PROXY:
                 _rem_obj = RemovedObject.objects.create(identifier=removed_id)
                 _rem_obj.save()
             remove_resource(_so_to_remove) 
             
     sys.stdout.write("\n{} resources removed\n".format(removed_count))
Beispiel #7
0
    def sync_with_nodes(nodes, is_proxy):
        """
        Synchronizes this META-SHARE node with the given other META-SHARE nodes.
        
        `nodes` is a dict of dicts with synchronization settings for the nodes
            to synchronize with
        `is_proxy` must be True if this node is a proxy for the given nodes;
            it must be False if the given nodes are not proxied by this node
        """
        for server in nodes.values():
            new_resources = []
            resources_to_update = []
            local_inventory = []

            # Login
            url = server['URL']
            user_name = server['USERNAME']
            password = server['PASSWORD']
            opener = login("{0}/login/".format(url), user_name, password)
            
            # Get the inventory list. 
            remote_inventory = get_inventory(opener, "{0}/sync/".format(url))
            remote_inventory_count = len(remote_inventory)
            sys.stdout.write("\nRemote node " + BOLD + url + RESET + " contains " + BOLD + str(remote_inventory_count) + " resources.\n" + RESET)
            
            # Get a list of uuid's and digests from the local inventory
            non_master_storage_objects = StorageObject.objects.exclude(copy_status=MASTER)
            for item in non_master_storage_objects:
                local_inventory.append({'id':str(item.identifier), 'digest':str(item.digest_checksum)})
            local_inventory_count = len(local_inventory)
            sys.stdout.write("\nLocal node contains " + BOLD + str(local_inventory_count) + " resources.\n" + RESET)

            # Create an list of ids to speed-up matching
            local_inventory_indexed = []
            for item in local_inventory:
                local_inventory_indexed.append(item['id'])
            
            # Create two lists:
            # 1. Containing items to be added - items that exist in the 
            # remote inventory and not in the local.
            # 2. Containing items to be updated - items that exist in both
            # inventories but the remote is different from the local
            for item in remote_inventory:
                item_id = item['id']
                if item_id not in local_inventory_indexed:
                    new_resources.append(item)
                else:
                    # Find the corresponding item in the local inventory
                    # and compare digests
                    for local_item in local_inventory:
                        if (item_id == local_item['id']) \
                          and not (item['digest'] == local_item['digest']):
                            resources_to_update.append(item)

 
            # Print informative messages to the user
            new_resources_count = len(new_resources)
            resources_to_update_count = len(resources_to_update)            
            if ((new_resources_count == 0) and (resources_to_update_count == 0)):
                sys.stdout.write("\nThere are no resources marked" +\
                  " for updating!\n")
            else:
				# If there are resources to add or update
                sys.stdout.write("\n" + BOLD + \
                  ("No" if new_resources_count == 0 \
                  else str(new_resources_count)) + \
                  " new resource" + ("" if new_resources_count == 1 else "s") \
                  + RESET + " will be added to your repository.\n")
                sys.stdout.write("\n" + BOLD + \
                  ("No" if resources_to_update_count == 0 \
                  else str(resources_to_update_count)) + \
                  " resource" + ("" if resources_to_update_count == 1 else "s") \
                  + RESET + " will be updated in your repository.\n")
                sys.stdout.write("\nImporting and Indexing...\n")

                if is_proxy:
                    _copy_status = PROXY
                else:
                    _copy_status = REMOTE
                
                # Get the full xmls from remore inventory and update local inventory
                for resource in new_resources:
                    # Get the json storage object and the actual metadata xml
                    storage_json, resource_xml_string = \
                      get_full_metadata(opener, "{0}/sync/{1}/metadata/" \
                            .format(url, resource['id']), resource['digest'])
                    update_resource(storage_json, resource_xml_string,
                                    resource['digest'], _copy_status)

                for resource in resources_to_update:
                    # Get the json storage object and the actual metadata xml
                    storage_json, resource_xml_string = \
                      get_full_metadata(opener, "{0}/sync/{1}/metadata/" \
                            .format(url, resource['id']), resource['digest'])
                    update_resource(storage_json, resource_xml_string,
                                    resource['digest'], _copy_status)

            sys.stdout.write("\n\n")