def test_update_new(self): """ Simulate the case where synchronization brings a new storage object to be instantiated. """ # Exercise update_resource(self.storage_json, self.metadata_before, self.storage_digest) # Verify self.assertEquals(1, StorageObject.objects.filter(identifier=self.storage_id).count()) storage_object = StorageObject.objects.get(identifier=self.storage_id) self.assertEquals(self.metadata_before, storage_object.metadata)
def test_master_copy(self): """ Verify that reusable entities such as persons have copy status MASTER after synchronization. """ # exercise update_resource(self.storage_json, self.metadata_before, None, copy_status=MASTER) # verify resource = resourceInfoType_model.objects.get(storage_object__identifier=self.storage_id) persons = resource.contactPerson.all() self.assertEquals(1, len(persons)) contact_person = persons[0] self.assertEquals(MASTER, contact_person.copy_status)
def test_update_refuse_mastercopy(self): """ Refuse to replace a master copy with a non-master copy during update """ # setup update_resource(self.storage_json, self.metadata_before, self.storage_digest, MASTER) self.assertEquals(MASTER, StorageObject.objects.get(identifier=self.storage_id).copy_status) # exercise try: update_resource(self.storage_json, self.metadata_modified, self.storage_digest, REMOTE) self.fail("Should have raised an exception") except IllegalAccessException: pass # Expected exception
def test_update_existing(self): """ Simulate update for already existing storage object """ # helper def get_metadatacreationdate_for(storage_id): resource = resourceInfoType_model.objects.get(storage_object__identifier=self.storage_id) return resource.metadataInfo.metadataCreationDate # setup update_resource(self.storage_json, self.metadata_before, self.storage_digest) self.assertEquals(date(2005, 5, 12), get_metadatacreationdate_for(self.storage_id)) self.assertEquals(REMOTE, StorageObject.objects.get(identifier=self.storage_id).copy_status) # exercise update_resource(self.storage_json, self.metadata_modified, self.storage_digest) self.assertEquals(date(2006, 12, 31), get_metadatacreationdate_for(self.storage_id))
def handle(self, *args, **options): # Get the list of the servers to be querried core_nodes = CORE_NODES #print core_nodes for server in core_nodes.values(): new_resources = [] resources_to_update = [] local_inventory = [] # Login url = server['URL'] user_name = server['USERNAME'] password = server['PASSWORD'] opener = login("{0}/login/".format(url), user_name, password) # Get the inventory list. remote_inventory = get_inventory(opener, "{0}/sync/".format(url)) remote_inventory_count = len(remote_inventory) sys.stdout.write("\nRemote node " + BOLD + url + RESET + " contains " + BOLD + str(remote_inventory_count) + " resources.\n" + RESET) # Get a list of uuid's and digests from the local inventory non_master_storage_objects = StorageObject.objects.exclude(copy_status=MASTER) for item in non_master_storage_objects: local_inventory.append({'id':str(item.identifier), 'digest':str(item.digest_checksum)}) #print "\nLOCAL INVENTORY: \n" + str(local_inventory) local_inventory_count = len(local_inventory) sys.stdout.write("\nLocal node contains " + BOLD + str(local_inventory_count) + " resources.\n" + RESET) # Create an list of ids to speed-up matching local_inventory_indexed = [] for item in local_inventory: local_inventory_indexed.append(item['id']) #print "\nINVENTORY LIST : \n" + str(local_inventory_indexed) # Create two lists: # 1. Containing items to be added - items that exist in the # remote inventory and not in the local. # 2. Containing items to be updated - items that exist in both # inventories but the remote is different from the local for item in remote_inventory: item_id = item['id'] if item_id not in local_inventory_indexed: new_resources.append(item) else: # Find the corresponding item in the local inventory # and compare digests for local_item in local_inventory: if (item_id == local_item['id']) \ and not (item['digest'] == local_item['digest']): resources_to_update.append(item) new_resources_count = len(new_resources) resources_to_update_count = len(resources_to_update) print new_resources sys.stdout.write("\n " + BOLD + str(new_resources_count) + \ " new resources" + RESET + " will be added to your repository.\n") sys.stdout.write("\n " + BOLD + str(resources_to_update_count) + \ " resources" + RESET + " will be updated in your repository.\n") sys.stdout.write("\n Importing and Indexing...\n") for resource in new_resources: # Get the json storage object and the actual metadata xml storage_json, resource_xml_string = get_full_metadata(opener, "{0}/sync/{1}/metadata/".format(url, resource['id']), resource['digest']) update_resource(storage_json, resource_xml_string) for resource in resources_to_update: # Get the json storage object and the actual metadata xml storage_json, resource_xml_string = get_full_metadata(opener, "{0}/sync/{1}/metadata/".format(url, resource['id']), resource['digest']) update_resource(storage_json, resource_xml_string)
def sync_with_server(server, is_proxy, id_file=None): """ Synchronizes this META-SHARE node with another META-SHARE node using the given server description. `nodes` is a dict of dicts with synchronization settings for the nodes to synchronize with `is_proxy` must be True if this node is a proxy for the given nodes; it must be False if the given nodes are not proxied by this node """ new_resources = [] resources_to_update = [] local_inventory = [] # Login url = server['URL'] user_name = server['USERNAME'] password = server['PASSWORD'] opener = login("{0}/login/".format(url), user_name, password) # Get the inventory list. remote_inventory = get_inventory(opener, "{0}/sync/".format(url)) # handle existing resources remote_inventory_existing = remote_inventory['existing'] remote_inventory_count = len(remote_inventory_existing) sys.stdout.write("\nRemote node " + BOLD + url + RESET + " contains " \ + BOLD + str(remote_inventory_count) + " resources.\n" + RESET) # Get a list of uuid's and digests from the local inventory non_master_storage_objects = StorageObject.objects.exclude(copy_status=MASTER) for item in non_master_storage_objects: local_inventory.append({'id':str(item.identifier), 'digest':str(item.digest_checksum)}) local_inventory_count = len(local_inventory) sys.stdout.write("\nLocal node contains " + BOLD + str(local_inventory_count) \ + " resources.\n" + RESET) # Create an list of ids to speed-up matching local_inventory_indexed = [] for item in local_inventory: local_inventory_indexed.append(item['id']) # Create two lists: # 1. Containing items to be added - items that exist in the # remote inventory and not in the local. # 2. Containing items to be updated - items that exist in both # inventories but the remote is different from the local for item in remote_inventory_existing: item_id = item['id'] if item_id not in local_inventory_indexed: new_resources.append(item) else: # Find the corresponding item in the local inventory # and compare digests for local_item in local_inventory: if (item_id == local_item['id']) \ and not (item['digest'] == local_item['digest']): resources_to_update.append(item) # Print informative messages to the user new_resources_count = len(new_resources) resources_to_update_count = len(resources_to_update) if ((new_resources_count == 0) and (resources_to_update_count == 0)): sys.stdout.write("\nThere are no resources marked" +\ " for updating!\n") else: # If there are resources to add or update sys.stdout.write("\n" + BOLD + \ ("No" if new_resources_count == 0 \ else str(new_resources_count)) + \ " new resource" + ("" if new_resources_count == 1 else "s") \ + RESET + " will be added to your repository.\n") sys.stdout.write("\n" + BOLD + \ ("No" if resources_to_update_count == 0 \ else str(resources_to_update_count)) + \ " resource" + ("" if resources_to_update_count == 1 else "s") \ + RESET + " will be updated in your repository.\n") sys.stdout.write("\nImporting and Indexing...\n") if is_proxy: _copy_status = PROXY else: _copy_status = REMOTE # Get the full xmls from remore inventory and update local inventory for resource in new_resources: # Get the json storage object and the actual metadata xml storage_json, resource_xml_string = \ get_full_metadata(opener, "{0}/sync/{1}/metadata/" \ .format(url, resource['id']), resource['digest']) res_obj = update_resource(storage_json, resource_xml_string, resource['digest'], _copy_status) if not id_file is None: id_file.write("--->RESOURCE_ID:{0};STORAGE_IDENTIFIER:{1}\n"\ .format(res_obj.id, res_obj.storage_object.identifier)) for resource in resources_to_update: # Get the json storage object and the actual metadata xml storage_json, resource_xml_string = \ get_full_metadata(opener, "{0}/sync/{1}/metadata/" \ .format(url, resource['id']), resource['digest']) res_obj = update_resource(storage_json, resource_xml_string, resource['digest'], _copy_status) if not id_file is None: id_file.write("--->RESOURCE_ID:{0};STORAGE_IDENTIFIER:{1}\n"\ .format(res_obj.id, res_obj.storage_object.identifier)) if resource['digest'] != res_obj.storage_object.digest_checksum: id_file.write("Different digests!\n") sys.stdout.write("\n\n") # handle removed resources remote_inventory_removed = remote_inventory['removed'] remote_inventory_removed_count = len(remote_inventory_removed) sys.stdout.write("\nRemote node " + BOLD + url + RESET + " lists " \ + BOLD + str(remote_inventory_removed_count) + " resources as removed.\n" + RESET) removed_count = 0 for removed_id in remote_inventory_removed: if removed_id in local_inventory_indexed: # remove resource from this node; # if it is a PROXY copy, also create a corresponding removed # object, so that the removal is propagated to other # META-SHARE Managing Nodes (aka. inner nodes) sys.stdout.write("\nRemoving id {}...\n".format(removed_id)) removed_count += 1 _so_to_remove = StorageObject.objects.get(identifier=removed_id) if _so_to_remove.copy_status is PROXY: _rem_obj = RemovedObject.objects.create(identifier=removed_id) _rem_obj.save() remove_resource(_so_to_remove) sys.stdout.write("\n{} resources removed\n".format(removed_count))
def sync_with_nodes(nodes, is_proxy): """ Synchronizes this META-SHARE node with the given other META-SHARE nodes. `nodes` is a dict of dicts with synchronization settings for the nodes to synchronize with `is_proxy` must be True if this node is a proxy for the given nodes; it must be False if the given nodes are not proxied by this node """ for server in nodes.values(): new_resources = [] resources_to_update = [] local_inventory = [] # Login url = server['URL'] user_name = server['USERNAME'] password = server['PASSWORD'] opener = login("{0}/login/".format(url), user_name, password) # Get the inventory list. remote_inventory = get_inventory(opener, "{0}/sync/".format(url)) remote_inventory_count = len(remote_inventory) sys.stdout.write("\nRemote node " + BOLD + url + RESET + " contains " + BOLD + str(remote_inventory_count) + " resources.\n" + RESET) # Get a list of uuid's and digests from the local inventory non_master_storage_objects = StorageObject.objects.exclude(copy_status=MASTER) for item in non_master_storage_objects: local_inventory.append({'id':str(item.identifier), 'digest':str(item.digest_checksum)}) local_inventory_count = len(local_inventory) sys.stdout.write("\nLocal node contains " + BOLD + str(local_inventory_count) + " resources.\n" + RESET) # Create an list of ids to speed-up matching local_inventory_indexed = [] for item in local_inventory: local_inventory_indexed.append(item['id']) # Create two lists: # 1. Containing items to be added - items that exist in the # remote inventory and not in the local. # 2. Containing items to be updated - items that exist in both # inventories but the remote is different from the local for item in remote_inventory: item_id = item['id'] if item_id not in local_inventory_indexed: new_resources.append(item) else: # Find the corresponding item in the local inventory # and compare digests for local_item in local_inventory: if (item_id == local_item['id']) \ and not (item['digest'] == local_item['digest']): resources_to_update.append(item) # Print informative messages to the user new_resources_count = len(new_resources) resources_to_update_count = len(resources_to_update) if ((new_resources_count == 0) and (resources_to_update_count == 0)): sys.stdout.write("\nThere are no resources marked" +\ " for updating!\n") else: # If there are resources to add or update sys.stdout.write("\n" + BOLD + \ ("No" if new_resources_count == 0 \ else str(new_resources_count)) + \ " new resource" + ("" if new_resources_count == 1 else "s") \ + RESET + " will be added to your repository.\n") sys.stdout.write("\n" + BOLD + \ ("No" if resources_to_update_count == 0 \ else str(resources_to_update_count)) + \ " resource" + ("" if resources_to_update_count == 1 else "s") \ + RESET + " will be updated in your repository.\n") sys.stdout.write("\nImporting and Indexing...\n") if is_proxy: _copy_status = PROXY else: _copy_status = REMOTE # Get the full xmls from remore inventory and update local inventory for resource in new_resources: # Get the json storage object and the actual metadata xml storage_json, resource_xml_string = \ get_full_metadata(opener, "{0}/sync/{1}/metadata/" \ .format(url, resource['id']), resource['digest']) update_resource(storage_json, resource_xml_string, resource['digest'], _copy_status) for resource in resources_to_update: # Get the json storage object and the actual metadata xml storage_json, resource_xml_string = \ get_full_metadata(opener, "{0}/sync/{1}/metadata/" \ .format(url, resource['id']), resource['digest']) update_resource(storage_json, resource_xml_string, resource['digest'], _copy_status) sys.stdout.write("\n\n")