Ejemplo n.º 1
0
    def read_reference_sitemap(self, ref_sitemap, name='reference'):
        """Read reference sitemap and return the inventory

        name parameter just uses in output messages to say what type
        of sitemap is being read.
        """
        sitemap = Sitemap(allow_multifile=self.allow_multifile,
                          mapper=self.mapper)
        self.logger.info("Reading %s sitemap(s) from %s ..." %
                         (name, ref_sitemap))
        i = sitemap.read(ref_sitemap)
        num_entries = len(i)
        self.logger.warning("Read %s sitemap with %d entries in %d sitemaps" %
                            (name, num_entries, sitemap.sitemaps_created))
        if (self.verbose):
            to_show = 100
            override_str = ' (override with --max-sitemap-entries)'
            if (self.max_sitemap_entries):
                to_show = self.max_sitemap_entries
                override_str = ''
            if (num_entries > to_show):
                print "Showing first %d entries sorted by URI%s..." % (
                    to_show, override_str)
            n = 0
            for r in i:
                print r
                n += 1
                if (n >= to_show):
                    break
        return (i)
Ejemplo n.º 2
0
    def read_reference_sitemap(self, ref_sitemap, name="reference"):
        """Read reference sitemap and return the inventory

        name parameter just uses in output messages to say what type
        of sitemap is being read.
        """
        sitemap = Sitemap(allow_multifile=self.allow_multifile, mapper=self.mapper)
        self.logger.info("Reading %s sitemap(s) from %s ..." % (name, ref_sitemap))
        i = sitemap.read(ref_sitemap)
        num_entries = len(i)
        self.logger.warning(
            "Read %s sitemap with %d entries in %d sitemaps" % (name, num_entries, sitemap.sitemaps_created)
        )
        if self.verbose:
            to_show = 100
            override_str = " (override with --max-sitemap-entries)"
            if self.max_sitemap_entries:
                to_show = self.max_sitemap_entries
                override_str = ""
            if num_entries > to_show:
                print "Showing first %d entries sorted by URI%s..." % (to_show, override_str)
            n = 0
            for r in i:
                print r
                n += 1
                if n >= to_show:
                    break
        return i
Ejemplo n.º 3
0
 def explore_links_get(self, uri, seen=[]):
     # Check we haven't been here before
     if uri in seen:
         self.logger.warning("Already see %s, skipping" % (uri))
     s = Sitemap(allow_multifile=self.allow_multifile)
     self.logger.info("Reading sitemap from %s ..." % (uri))
     i = s.read(uri, index_only=True)
     self.logger.warning("Read %s from %s" % (s.read_type, uri))
     links = self.extract_links(i, verbose=True)
     if "next" in links and links["next"] == uri:
         self.logger.warning('- self reference "next" link')
     seen[uri] = links
     return (s.changeset_read, links)
Ejemplo n.º 4
0
 def explore_links_get(self, uri, seen=[]):
     # Check we haven't been here before
     if (uri in seen):
         self.logger.warning("Already see %s, skipping" % (uri))
     s = Sitemap(allow_multifile=self.allow_multifile)
     self.logger.info("Reading sitemap from %s ..." % (uri))
     i = s.read(uri, index_only=True)
     self.logger.warning("Read %s from %s" % (s.read_type, uri))
     links = self.extract_links(i, verbose=True)
     if ('next' in links and links['next'] == uri):
         self.logger.warning("- self reference \"next\" link")
     seen[uri] = links
     return (s.changeset_read, links)
Ejemplo n.º 5
0
 def changeset_sitemap(self,
                       outfile=None,
                       ref_sitemap=None,
                       capabilities=None,
                       dump=None):
     # 1. Get and parse reference sitemap
     rs = Sitemap(verbose=self.verbose,
                  allow_multifile=self.allow_multifile,
                  mapper=self.mapper)
     if (self.verbose):
         print "Reading sitemap(s) from %s ..." % (ref_sitemap)
     ri = rs.read(ref_sitemap)
     num_entries = len(ri)
     print "Read reference sitemap with %d entries in %d sitemaps" % (
         num_entries, rs.sitemaps_created)
     if (self.verbose):
         to_show = 100
         override_str = ' (override with --max-sitemap-entries)'
         if (self.max_sitemap_entries):
             to_show = self.max_sitemap_entries
             override_str = ''
         if (num_entries > to_show):
             print "Showing first %d entries sorted by URI%s..." % (
                 to_show, override_str)
         n = 0
         for r in ri.resource_uris():
             print ri.resources[r]
             n += 1
             if (n >= to_show):
                 break
     # 2. Set up base_path->base_uri mappings, get inventory from disk
     disk_inventory = self.inventory
     # 3. Calculate changeset
     (num_same, updated, deleted, created) = ri.compare(disk_inventory)
     changeset = Inventory()
     changeset.capabilities = capabilities
     changeset.add(disk_inventory.changeset(updated, changetype='updated'))
     changeset.add(ri.changeset(deleted, changetype='deleted'))
     changeset.add(disk_inventory.changeset(created, changetype='created'))
     # 4. Write out changeset
     s = Sitemap(verbose=self.verbose,
                 pretty_xml=True,
                 allow_multifile=self.allow_multifile,
                 mapper=self.mapper)
     if (self.max_sitemap_entries is not None):
         s.max_sitemap_entries = self.max_sitemap_entries
     if (outfile is None):
         print s.inventory_as_xml(changeset)
     else:
         s.write(changeset, basename=outfile)
     self.write_dump_if_requested(changeset, dump)
Ejemplo n.º 6
0
    def changeset_sitemap(self,outfile=None,ref_sitemap=None,capabilities=None,
                          dump=None):
        # 1. Get and parse reference sitemap
        rs = Sitemap(verbose=self.verbose, allow_multifile=self.allow_multifile, 
                     mapper=self.mapper)
        if (self.verbose):
            print "Reading sitemap(s) from %s ..." % (ref_sitemap)
        ri = rs.read(ref_sitemap)
        num_entries = len(ri)
        print "Read reference sitemap with %d entries in %d sitemaps" % (num_entries,rs.sitemaps_created)
        if (self.verbose):
            to_show = 100
            override_str = ' (override with --max-sitemap-entries)'
            if (self.max_sitemap_entries):
                to_show = self.max_sitemap_entries
                override_str = ''
            if (num_entries>to_show):
                print "Showing first %d entries sorted by URI%s..." % (to_show,override_str)
            n=0
            for r in ri.resource_uris():
                print ri.resources[r]
                n+=1
                if ( n >= to_show ):
                    break
        # 2. Set up base_path->base_uri mappings, get inventory from disk
        disk_inventory = self.inventory
        # 3. Calculate changeset
        (num_same,updated,deleted,created)=ri.compare(disk_inventory)   
        changeset = Inventory()
        changeset.capabilities = capabilities
        changeset.add( disk_inventory.changeset( updated, changetype='updated' ) )
        changeset.add( ri.changeset( deleted, changetype='deleted' ) )
        changeset.add( disk_inventory.changeset( created, changetype='created' ) )
        # 4. Write out changeset
        s = Sitemap(verbose=self.verbose, pretty_xml=True, allow_multifile=self.allow_multifile,
	            mapper=self.mapper)
        if (self.max_sitemap_entries is not None):
            s.max_sitemap_entries = self.max_sitemap_entries
        if (outfile is None):
            print s.inventory_as_xml(changeset)
        else:
            s.write(changeset,basename=outfile)
        self.write_dump_if_requested(changeset,dump)
Ejemplo n.º 7
0
 def parse_sitemap(self):
     s = Sitemap(allow_multifile=self.allow_multifile)
     self.logger.info("Reading sitemap(s) from %s ..." % (self.sitemap))
     i = s.read(self.sitemap)
     num_entries = len(i)
     self.logger.warning("Read sitemap with %d entries in %d sitemaps" % (num_entries, s.sitemaps_created))
     if self.verbose:
         to_show = 100
         override_str = " (override with --max-sitemap-entries)"
         if self.max_sitemap_entries:
             to_show = self.max_sitemap_entries
             override_str = ""
         if num_entries > to_show:
             print "Showing first %d entries sorted by URI%s..." % (to_show, override_str)
         n = 0
         for r in i:
             print r
             n += 1
             if n >= to_show:
                 break
Ejemplo n.º 8
0
 def parse_sitemap(self):
     s=Sitemap(verbose=self.verbose, allow_multifile=self.allow_multifile)
     if (self.verbose):
         print "Reading sitemap(s) from %s ..." % (sitemap)
     i = s.read(sitemap)
     num_entries = len(i)
     print "Read sitemap with %d entries in %d sitemaps" % (num_entries,s.sitemaps_created)
     if (self.verbose):
         to_show = 100
         override_str = ' (override with --max-sitemap-entries)'
         if (self.max_sitemap_entries):
             to_show = self.max_sitemap_entries
             override_str = ''
         if (num_entries>to_show):
             print "Showing first %d entries sorted by URI%s..." % (to_show,override_str)
         n=0
         for r in i.resource_uris():
             print i.resources[r]
             n+=1
             if ( n >= to_show ):
                 break
Ejemplo n.º 9
0
 def parse_sitemap(self):
     s = Sitemap(allow_multifile=self.allow_multifile)
     self.logger.info("Reading sitemap(s) from %s ..." % (self.sitemap))
     i = s.read(self.sitemap)
     num_entries = len(i)
     self.logger.warning("Read sitemap with %d entries in %d sitemaps" %
                         (num_entries, s.sitemaps_created))
     if (self.verbose):
         to_show = 100
         override_str = ' (override with --max-sitemap-entries)'
         if (self.max_sitemap_entries):
             to_show = self.max_sitemap_entries
             override_str = ''
         if (num_entries > to_show):
             print "Showing first %d entries sorted by URI%s..." % (
                 to_show, override_str)
         n = 0
         for r in i:
             print r
             n += 1
             if (n >= to_show):
                 break
Ejemplo n.º 10
0
 def parse_sitemap(self):
     s = Sitemap(verbose=self.verbose, allow_multifile=self.allow_multifile)
     if (self.verbose):
         print "Reading sitemap(s) from %s ..." % (sitemap)
     i = s.read(sitemap)
     num_entries = len(i)
     print "Read sitemap with %d entries in %d sitemaps" % (
         num_entries, s.sitemaps_created)
     if (self.verbose):
         to_show = 100
         override_str = ' (override with --max-sitemap-entries)'
         if (self.max_sitemap_entries):
             to_show = self.max_sitemap_entries
             override_str = ''
         if (num_entries > to_show):
             print "Showing first %d entries sorted by URI%s..." % (
                 to_show, override_str)
         n = 0
         for r in i.resource_uris():
             print i.resources[r]
             n += 1
             if (n >= to_show):
                 break
Ejemplo n.º 11
0
    def incremental(self, allow_deletion=False, changelist_uri=None):
	"""Incremental synchronization"""
        self.logger.debug("Starting incremental sync")
        ### 0. Sanity checks
        if (len(self.mappings)<1):
            raise ClientFatalError("No source to destination mapping specified")
        # Get current config
        inc_config_next=self.read_incremental_status(self.sitemap)
        ### 1. Get URI of changelist, from sitemap or explicit
        if (inc_config_next is not None):
            # We have config from last run for this site
            changelist = inc_config_next
            self.logger.info("ChangeList location from last incremental run %s" % (changelist))
        elif (changelist_uri):
            # Translate as necessary using maps
            changelist = self.sitemap_changelist_uri(changelist_uri)
        else:
            # Get sitemap
            try:
                self.logger.info("Reading sitemap %s" % (self.sitemap))
                src_sitemap = Sitemap(allow_multifile=self.allow_multifile, mapper=self.mapper)
                src_resourcelist = src_sitemap.read(uri=self.sitemap, index_only=True)
                self.logger.debug("Finished reading sitemap/sitemapindex")
            except Exception as e:
                raise ClientFatalError("Can't read source sitemap from %s (%s)" % (self.sitemap,str(e)))
            # Extract changelist location
            # FIXME - need to completely rework the way we handle/store capabilities
            links = self.extract_links(src_resourcelist)
            if ('current' not in links):
                raise ClientFatalError("Failed to extract changelist location from sitemap %s" % (self.sitemap))
            changelist = links['current']
        ### 2. Read changelist from source
        ib = ResourceListBuilder(mapper=self.mapper)
        try:
            self.logger.info("Reading changelist %s" % (changelist))
            src_sitemap = Sitemap(allow_multifile=self.allow_multifile, mapper=self.mapper)
            src_changelist = src_sitemap.read(uri=changelist, changelist=True)
            self.logger.debug("Finished reading changelist")
        except Exception as e:
            raise ClientFatalError("Can't read source changelist from %s (%s)" % (changelist,str(e)))
        self.logger.info("Read source changelist, %d resources listed" % (len(src_changelist)))
        #if (len(src_changelist)==0):
        #    raise ClientFatalError("Aborting as there are no resources to sync")
        if (self.checksum and not src_changelist.has_md5()):
            self.checksum=False
            self.logger.info("Not calculating checksums on destination as not present in source resourcelist")
        ### 3. Check that sitemap has authority over URIs listed
        # FIXME - What does authority mean for changelist? Here use both the
        # changelist URI and, if we used it, the sitemap URI
        uauth_cs = UrlAuthority(changelist)
        if (not changelist_uri):
            uauth_sm = UrlAuthority(self.sitemap)
        for resource in src_changelist:
            if (not uauth_cs.has_authority_over(resource.uri) and 
                (changelist_uri or not uauth_sm.has_authority_over(resource.uri))):
                if (self.noauth):
                    #self.logger.info("ChangeList (%s) mentions resource at a location it does not have authority over (%s)" % (changelist,resource.uri))
                    pass
                else:
                    raise ClientFatalError("Aborting as changelist (%s) mentions resource at a location it does not have authority over (%s), override with --noauth" % (changelist,resource.uri))
        ### 3. Apply changes
        num_updated = 0
        num_deleted = 0
        num_created = 0
        for resource in src_changelist:
            uri = resource.uri
            file = self.mapper.src_to_dst(uri)
            if (resource.change == 'updated'):
                self.logger.info("updated: %s -> %s" % (uri,file))
                self.update_resource(resource,file,'updated')
                num_updated+=1
            elif (resource.change == 'created'):
                self.logger.info("created: %s -> %s" % (uri,file))
                self.update_resource(resource,file,'created')
                num_created+=1
            elif (resource.change == 'deleted'):
                self.delete_resource(resource,file,allow_deletion)
                num_deleted+=1
            else:
                raise ClientError("Unknown change type %s" % (resource.change) )
        # 4. Report status and planned actions
        status = "NO CHANGES"
        if ((num_updated+num_deleted+num_created)>0):
            status = " CHANGES  "
        self.logger.warning("Status: %s (updated=%d, deleted=%d, created=%d)" %\
              (status,num_updated,num_deleted,num_created))
        # 5. Store next link if available
        if ((num_updated+num_deleted+num_created)>0):
            links = self.extract_links(src_changelist)
            if ('next' in links):
                self.write_incremental_status(self.sitemap,links['next'])
                self.logger.info("Written config with next incremental at %s" % (links['next']))
            else:
                self.logger.warning("Failed to extract next changelist location from changelist %s" % (changelist))
        # 6. Done
        self.logger.debug("Completed incremental sync")
Ejemplo n.º 12
0
 def sync_or_audit(self, allow_deletion=False, audit_only=False):
     action = "audit" if (audit_only) else "sync"
     self.logger.debug("Starting " + action)
     ### 0. Sanity checks
     if len(self.mappings) < 1:
         raise ClientFatalError("No source to destination mapping specified")
     ### 1. Get inventories from both src and dst
     # 1.a source inventory
     ib = InventoryBuilder(mapper=self.mapper)
     try:
         self.logger.info("Reading sitemap %s" % (self.sitemap))
         src_sitemap = Sitemap(allow_multifile=self.allow_multifile, mapper=self.mapper)
         src_inventory = src_sitemap.read(uri=self.sitemap)
         self.logger.debug("Finished reading sitemap")
     except Exception as e:
         raise ClientFatalError("Can't read source inventory from %s (%s)" % (self.sitemap, str(e)))
     self.logger.info("Read source inventory, %d resources listed" % (len(src_inventory)))
     if len(src_inventory) == 0:
         raise ClientFatalError("Aborting as there are no resources to sync")
     if self.checksum and not src_inventory.has_md5():
         self.checksum = False
         self.logger.info("Not calculating checksums on destination as not present in source inventory")
     # 1.b destination inventory mapped back to source URIs
     ib.do_md5 = self.checksum
     dst_inventory = ib.from_disk()
     ### 2. Compare these inventorys respecting any comparison options
     (same, updated, deleted, created) = dst_inventory.compare(src_inventory)
     ### 3. Report status and planned actions
     status = "  IN SYNC  "
     if len(updated) > 0 or len(deleted) > 0 or len(created) > 0:
         status = "NOT IN SYNC"
     self.logger.warning(
         "Status: %s (same=%d, updated=%d, deleted=%d, created=%d)"
         % (status, len(same), len(updated), len(deleted), len(created))
     )
     if audit_only:
         self.logger.debug("Completed " + action)
         return
     ### 4. Check that sitemap has authority over URIs listed
     uauth = UrlAuthority(self.sitemap)
     for resource in src_inventory:
         if not uauth.has_authority_over(resource.uri):
             if self.noauth:
                 self.logger.info(
                     "Sitemap (%s) mentions resource at a location it does not have authority over (%s)"
                     % (self.sitemap, resource.uri)
                 )
             else:
                 raise ClientFatalError(
                     "Aborting as sitemap (%s) mentions resource at a location it does not have authority over (%s), override with --noauth"
                     % (self.sitemap, resource.uri)
                 )
     ### 5. Grab files to do sync
     for resource in updated:
         uri = resource.uri
         file = self.mapper.src_to_dst(uri)
         self.logger.info("updated: %s -> %s" % (uri, file))
         self.update_resource(resource, file, "UPDATED")
     for resource in created:
         uri = resource.uri
         file = self.mapper.src_to_dst(uri)
         self.logger.info("created: %s -> %s" % (uri, file))
         self.update_resource(resource, file, "CREATED")
     for resource in deleted:
         uri = resource.uri
         if allow_deletion:
             file = self.mapper.src_to_dst(uri)
             if self.dryrun:
                 self.logger.info("dryrun: would delete %s -> %s" % (uri, file))
             else:
                 os.unlink(file)
                 self.logger.info("deleted: %s -> %s" % (uri, file))
                 self.log_event(ResourceChange(resource=resource, changetype="DELETED"))
         else:
             self.logger.info("nodelete: would delete %s (--delete to enable)" % uri)
     self.logger.debug("Completed " + action)
Ejemplo n.º 13
0
 def incremental(self, allow_deletion=False, changeset_uri=None):
     self.logger.debug("Starting incremental sync")
     ### 0. Sanity checks
     if (len(self.mappings) < 1):
         raise ClientFatalError(
             "No source to destination mapping specified")
     ### 1. Get URI of changeset, from sitemap or explicit
     if (changeset_uri):
         # Translate as necessary using maps
         changeset = self.sitemap_changeset_uri(changeset_uri)
     else:
         # Get sitemap
         try:
             self.logger.info("Reading sitemap %s" % (self.sitemap))
             src_sitemap = Sitemap(allow_multifile=self.allow_multifile,
                                   mapper=self.mapper)
             src_inventory = src_sitemap.read(uri=self.sitemap,
                                              index_only=True)
             self.logger.debug("Finished reading sitemap/sitemapindex")
         except Exception as e:
             raise ClientFatalError(
                 "Can't read source sitemap from %s (%s)" %
                 (self.sitemap, str(e)))
         # Extract changeset location
         # FIXME - need to completely rework the way we handle/store capabilities
         links = self.extract_links(src_inventory.capabilities)
         if ('current' not in links):
             raise ClientFatalError(
                 "Failed to extract changeset location from sitemap %s" %
                 (self.sitemap))
         changeset = links['current']
     ### 2. Read changeset from source
     ib = InventoryBuilder(mapper=self.mapper)
     try:
         self.logger.info("Reading changeset %s" % (changeset))
         src_sitemap = Sitemap(allow_multifile=self.allow_multifile,
                               mapper=self.mapper)
         src_changeset = src_sitemap.read(uri=changeset, changeset=True)
         self.logger.debug("Finished reading changeset")
     except Exception as e:
         raise ClientFatalError("Can't read source changeset from %s (%s)" %
                                (changeset, str(e)))
     self.logger.info("Read source changeset, %d resources listed" %
                      (len(src_changeset)))
     if (len(src_changeset) == 0):
         raise ClientFatalError(
             "Aborting as there are no resources to sync")
     if (self.checksum and not src_changeset.has_md5()):
         self.checksum = False
         self.logger.info(
             "Not calculating checksums on destination as not present in source inventory"
         )
     ### 3. Check that sitemap has authority over URIs listed
     # FIXME - What does authority mean for changeset? Here use both the
     # changeset URI and, if we used it, the sitemap URI
     uauth_cs = UrlAuthority(changeset)
     if (not changeset_uri):
         uauth_sm = UrlAuthority(self.sitemap)
     for resource in src_changeset:
         if (not uauth_cs.has_authority_over(resource.uri)
                 and (changeset_uri
                      or not uauth_sm.has_authority_over(resource.uri))):
             if (self.noauth):
                 self.logger.warning(
                     "Changeset (%s) mentions resource at a location it does not have authority over (%s)"
                     % (changeset, resource.uri))
             else:
                 raise ClientFatalError(
                     "Aborting as changeset (%s) mentions resource at a location it does not have authority over (%s), override with --noauth"
                     % (changeset, resource.uri))
     ### 3. Apply changes
     for resource in src_changeset:
         uri = resource.uri
         file = self.mapper.src_to_dst(uri)
         if (resource.changetype == 'UPDATED'):
             self.logger.info("updated: %s -> %s" % (uri, file))
             self.update_resource(resource, file, 'UPDATED')
         elif (resource.changetype == 'CREATED'):
             self.logger.info("created: %s -> %s" % (uri, file))
             self.update_resource(resource, file, 'CREATED')
         elif (resource.changetype == 'DELETED'):
             if (allow_deletion):
                 file = self.mapper.src_to_dst(uri)
                 if (self.dryrun):
                     self.logger.info("dryrun: would delete %s -> %s" %
                                      (uri, file))
                 else:
                     os.unlink(file)
                     self.logger.info("deleted: %s -> %s" % (uri, file))
                     self.log_event(
                         ResourceChange(resource=resource,
                                        changetype="DELETED"))
             else:
                 self.logger.info(
                     "nodelete: would delete %s (--delete to enable)" % uri)
         else:
             raise ClientError("Unknown change type %s" %
                               (resource.changetype))
     self.logger.debug("Completed incremental stuff")
Ejemplo n.º 14
0
 def sync_or_audit(self, allow_deletion=False, audit_only=False):
     action = ('audit' if (audit_only) else 'sync')
     self.logger.debug("Starting " + action)
     ### 0. Sanity checks
     if (len(self.mappings) < 1):
         raise ClientFatalError(
             "No source to destination mapping specified")
     ### 1. Get inventories from both src and dst
     # 1.a source inventory
     ib = InventoryBuilder(mapper=self.mapper)
     try:
         self.logger.info("Reading sitemap %s" % (self.sitemap))
         src_sitemap = Sitemap(allow_multifile=self.allow_multifile,
                               mapper=self.mapper)
         src_inventory = src_sitemap.read(uri=self.sitemap)
         self.logger.debug("Finished reading sitemap")
     except Exception as e:
         raise ClientFatalError("Can't read source inventory from %s (%s)" %
                                (self.sitemap, str(e)))
     self.logger.info("Read source inventory, %d resources listed" %
                      (len(src_inventory)))
     if (len(src_inventory) == 0):
         raise ClientFatalError(
             "Aborting as there are no resources to sync")
     if (self.checksum and not src_inventory.has_md5()):
         self.checksum = False
         self.logger.info(
             "Not calculating checksums on destination as not present in source inventory"
         )
     # 1.b destination inventory mapped back to source URIs
     ib.do_md5 = self.checksum
     dst_inventory = ib.from_disk()
     ### 2. Compare these inventorys respecting any comparison options
     (same, updated, deleted,
      created) = dst_inventory.compare(src_inventory)
     ### 3. Report status and planned actions
     status = "  IN SYNC  "
     if (len(updated) > 0 or len(deleted) > 0 or len(created) > 0):
         status = "NOT IN SYNC"
     self.logger.warning("Status: %s (same=%d, updated=%d, deleted=%d, created=%d)" %\
           (status,len(same),len(updated),len(deleted),len(created)))
     if (audit_only):
         self.logger.debug("Completed " + action)
         return
     ### 4. Check that sitemap has authority over URIs listed
     uauth = UrlAuthority(self.sitemap)
     for resource in src_inventory:
         if (not uauth.has_authority_over(resource.uri)):
             if (self.noauth):
                 self.logger.info(
                     "Sitemap (%s) mentions resource at a location it does not have authority over (%s)"
                     % (self.sitemap, resource.uri))
             else:
                 raise ClientFatalError(
                     "Aborting as sitemap (%s) mentions resource at a location it does not have authority over (%s), override with --noauth"
                     % (self.sitemap, resource.uri))
     ### 5. Grab files to do sync
     for resource in updated:
         uri = resource.uri
         file = self.mapper.src_to_dst(uri)
         self.logger.info("updated: %s -> %s" % (uri, file))
         self.update_resource(resource, file, 'UPDATED')
     for resource in created:
         uri = resource.uri
         file = self.mapper.src_to_dst(uri)
         self.logger.info("created: %s -> %s" % (uri, file))
         self.update_resource(resource, file, 'CREATED')
     for resource in deleted:
         uri = resource.uri
         if (allow_deletion):
             file = self.mapper.src_to_dst(uri)
             if (self.dryrun):
                 self.logger.info("dryrun: would delete %s -> %s" %
                                  (uri, file))
             else:
                 os.unlink(file)
                 self.logger.info("deleted: %s -> %s" % (uri, file))
                 self.log_event(
                     ResourceChange(resource=resource,
                                    changetype="DELETED"))
         else:
             self.logger.info(
                 "nodelete: would delete %s (--delete to enable)" % uri)
     self.logger.debug("Completed " + action)
Ejemplo n.º 15
0
 def read_example(self,file,changeset=False):
     s = Sitemap()
     r = s.read('resync/test/testdata/examples_from_spec/'+file)
     self.assertEqual( s.changeset_read, changeset )
     return r
Ejemplo n.º 16
0
 def read_example(self, file, changeset=False):
     s = Sitemap()
     r = s.read('resync/test/testdata/examples_from_spec/' + file)
     self.assertEqual(s.changeset_read, changeset)
     return r
Ejemplo n.º 17
0
 def incremental(self, allow_deletion=False, changeset_uri=None):
     self.logger.debug("Starting incremental sync")
     ### 0. Sanity checks
     if len(self.mappings) < 1:
         raise ClientFatalError("No source to destination mapping specified")
     ### 1. Get URI of changeset, from sitemap or explicit
     if changeset_uri:
         # Translate as necessary using maps
         changeset = self.sitemap_changeset_uri(changeset_uri)
     else:
         # Get sitemap
         try:
             self.logger.info("Reading sitemap %s" % (self.sitemap))
             src_sitemap = Sitemap(allow_multifile=self.allow_multifile, mapper=self.mapper)
             src_inventory = src_sitemap.read(uri=self.sitemap, index_only=True)
             self.logger.debug("Finished reading sitemap/sitemapindex")
         except Exception as e:
             raise ClientFatalError("Can't read source sitemap from %s (%s)" % (self.sitemap, str(e)))
         # Extract changeset location
         # FIXME - need to completely rework the way we handle/store capabilities
         links = self.extract_links(src_inventory.capabilities)
         if "current" not in links:
             raise ClientFatalError("Failed to extract changeset location from sitemap %s" % (self.sitemap))
         changeset = links["current"]
     ### 2. Read changeset from source
     ib = InventoryBuilder(mapper=self.mapper)
     try:
         self.logger.info("Reading changeset %s" % (changeset))
         src_sitemap = Sitemap(allow_multifile=self.allow_multifile, mapper=self.mapper)
         src_changeset = src_sitemap.read(uri=changeset, changeset=True)
         self.logger.debug("Finished reading changeset")
     except Exception as e:
         raise ClientFatalError("Can't read source changeset from %s (%s)" % (changeset, str(e)))
     self.logger.info("Read source changeset, %d resources listed" % (len(src_changeset)))
     if len(src_changeset) == 0:
         raise ClientFatalError("Aborting as there are no resources to sync")
     if self.checksum and not src_changeset.has_md5():
         self.checksum = False
         self.logger.info("Not calculating checksums on destination as not present in source inventory")
     ### 3. Check that sitemap has authority over URIs listed
     # FIXME - What does authority mean for changeset? Here use both the
     # changeset URI and, if we used it, the sitemap URI
     uauth_cs = UrlAuthority(changeset)
     if not changeset_uri:
         uauth_sm = UrlAuthority(self.sitemap)
     for resource in src_changeset:
         if not uauth_cs.has_authority_over(resource.uri) and (
             changeset_uri or not uauth_sm.has_authority_over(resource.uri)
         ):
             if self.noauth:
                 self.logger.warning(
                     "Changeset (%s) mentions resource at a location it does not have authority over (%s)"
                     % (changeset, resource.uri)
                 )
             else:
                 raise ClientFatalError(
                     "Aborting as changeset (%s) mentions resource at a location it does not have authority over (%s), override with --noauth"
                     % (changeset, resource.uri)
                 )
     ### 3. Apply changes
     for resource in src_changeset:
         uri = resource.uri
         file = self.mapper.src_to_dst(uri)
         if resource.changetype == "UPDATED":
             self.logger.info("updated: %s -> %s" % (uri, file))
             self.update_resource(resource, file, "UPDATED")
         elif resource.changetype == "CREATED":
             self.logger.info("created: %s -> %s" % (uri, file))
             self.update_resource(resource, file, "CREATED")
         elif resource.changetype == "DELETED":
             if allow_deletion:
                 file = self.mapper.src_to_dst(uri)
                 if self.dryrun:
                     self.logger.info("dryrun: would delete %s -> %s" % (uri, file))
                 else:
                     os.unlink(file)
                     self.logger.info("deleted: %s -> %s" % (uri, file))
                     self.log_event(ResourceChange(resource=resource, changetype="DELETED"))
             else:
                 self.logger.info("nodelete: would delete %s (--delete to enable)" % uri)
         else:
             raise ClientError("Unknown change type %s" % (resource.changetype))
     self.logger.debug("Completed incremental stuff")
Ejemplo n.º 18
0
    def baseline_or_audit(self, allow_deletion=False, audit_only=False):
        """Baseline synchonization or audit

	Both functions implemented in this routine because audit is a prerequisite
	for a baseline sync.
	"""
        action = ( 'audit' if (audit_only) else 'baseline sync' ) 
        self.logger.debug("Starting "+action)
        ### 0. Sanity checks
        if (len(self.mappings)<1):
            raise ClientFatalError("No source to destination mapping specified")
        ### 1. Get inventories from both src and dst 
        # 1.a source resourcelist
        ib = ResourceListBuilder(mapper=self.mapper)
        try:
            self.logger.info("Reading sitemap %s" % (self.sitemap))
            src_sitemap = Sitemap(allow_multifile=self.allow_multifile, mapper=self.mapper)
            src_resourcelist = src_sitemap.read(uri=self.sitemap)
            self.logger.debug("Finished reading sitemap")
        except Exception as e:
            raise ClientFatalError("Can't read source resourcelist from %s (%s)" % (self.sitemap,str(e)))
        self.logger.info("Read source resourcelist, %d resources listed" % (len(src_resourcelist)))
        if (len(src_resourcelist)==0):
            raise ClientFatalError("Aborting as there are no resources to sync")
        if (self.checksum and not src_resourcelist.has_md5()):
            self.checksum=False
            self.logger.info("Not calculating checksums on destination as not present in source resourcelist")
        # 1.b destination resourcelist mapped back to source URIs
        ib.do_md5=self.checksum
        dst_resourcelist = ib.from_disk()
        ### 2. Compare these resourcelists respecting any comparison options
        (same,updated,deleted,created)=dst_resourcelist.compare(src_resourcelist)   
        ### 3. Report status and planned actions
        status = "  IN SYNC  "
        if (len(updated)>0 or len(deleted)>0 or len(created)>0):
            status = "NOT IN SYNC"
        self.logger.warning("Status: %s (same=%d, updated=%d, deleted=%d, created=%d)" %\
              (status,len(same),len(updated),len(deleted),len(created)))
        if (audit_only):
            self.logger.debug("Completed "+action)
            return
        ### 4. Check that sitemap has authority over URIs listed
        uauth = UrlAuthority(self.sitemap)
        for resource in src_resourcelist:
            if (not uauth.has_authority_over(resource.uri)):
                if (self.noauth):
                    #self.logger.info("Sitemap (%s) mentions resource at a location it does not have authority over (%s)" % (self.sitemap,resource.uri))
                    pass
                else:
                    raise ClientFatalError("Aborting as sitemap (%s) mentions resource at a location it does not have authority over (%s), override with --noauth" % (self.sitemap,resource.uri))
        ### 5. Grab files to do sync
        for resource in updated:
            uri = resource.uri
            file = self.mapper.src_to_dst(uri)
            self.logger.info("updated: %s -> %s" % (uri,file))
            self.update_resource(resource,file,'updated')
        for resource in created:
            uri = resource.uri
            file = self.mapper.src_to_dst(uri)
            self.logger.info("created: %s -> %s" % (uri,file))
            self.update_resource(resource,file,'created')
        for resource in deleted:
            uri = resource.uri
            file = self.mapper.src_to_dst(uri)
            self.delete_resource(resource,file,allow_deletion)
        ### 6. For sync reset any incremental status for site
        if (not audit_only):
            links = self.extract_links(src_resourcelist)
            if ('next' in links):
                self.write_incremental_status(self.sitemap,links['next'])
                self.logger.info("Written config with next incremental at %s" % (links['next']))
            else:
                self.write_incremental_status(self.sitemap)
        self.logger.debug("Completed "+action)