def read_reference_sitemap(self, ref_sitemap, name='reference'):
"""Read reference sitemap and return the inventory
name parameter just uses in output messages to say what type
of sitemap is being read.
"""
sitemap = Sitemap(allow_multifile=self.allow_multifile,
mapper=self.mapper)
self.logger.info("Reading %s sitemap(s) from %s ..." %
(name, ref_sitemap))
i = sitemap.read(ref_sitemap)
num_entries = len(i)
self.logger.warning("Read %s sitemap with %d entries in %d sitemaps" %
(name, num_entries, sitemap.sitemaps_created))
if (self.verbose):
to_show = 100
override_str = ' (override with --max-sitemap-entries)'
if (self.max_sitemap_entries):
to_show = self.max_sitemap_entries
override_str = ''
if (num_entries > to_show):
print "Showing first %d entries sorted by URI%s..." % (
to_show, override_str)
n = 0
for r in i:
print r
n += 1
if (n >= to_show):
break
return (i)
def read_reference_sitemap(self, ref_sitemap, name="reference"):
"""Read reference sitemap and return the inventory
name parameter just uses in output messages to say what type
of sitemap is being read.
"""
sitemap = Sitemap(allow_multifile=self.allow_multifile, mapper=self.mapper)
self.logger.info("Reading %s sitemap(s) from %s ..." % (name, ref_sitemap))
i = sitemap.read(ref_sitemap)
num_entries = len(i)
self.logger.warning(
"Read %s sitemap with %d entries in %d sitemaps" % (name, num_entries, sitemap.sitemaps_created)
)
if self.verbose:
to_show = 100
override_str = " (override with --max-sitemap-entries)"
if self.max_sitemap_entries:
to_show = self.max_sitemap_entries
override_str = ""
if num_entries > to_show:
print "Showing first %d entries sorted by URI%s..." % (to_show, override_str)
n = 0
for r in i:
print r
n += 1
if n >= to_show:
break
return i
def explore_links_get(self, uri, seen=[]):
# Check we haven't been here before
if uri in seen:
self.logger.warning("Already see %s, skipping" % (uri))
s = Sitemap(allow_multifile=self.allow_multifile)
self.logger.info("Reading sitemap from %s ..." % (uri))
i = s.read(uri, index_only=True)
self.logger.warning("Read %s from %s" % (s.read_type, uri))
links = self.extract_links(i, verbose=True)
if "next" in links and links["next"] == uri:
self.logger.warning('- self reference "next" link')
seen[uri] = links
return (s.changeset_read, links)
def explore_links_get(self, uri, seen=[]):
# Check we haven't been here before
if (uri in seen):
self.logger.warning("Already see %s, skipping" % (uri))
s = Sitemap(allow_multifile=self.allow_multifile)
self.logger.info("Reading sitemap from %s ..." % (uri))
i = s.read(uri, index_only=True)
self.logger.warning("Read %s from %s" % (s.read_type, uri))
links = self.extract_links(i, verbose=True)
if ('next' in links and links['next'] == uri):
self.logger.warning("- self reference \"next\" link")
seen[uri] = links
return (s.changeset_read, links)
def changeset_sitemap(self,
outfile=None,
ref_sitemap=None,
capabilities=None,
dump=None):
# 1. Get and parse reference sitemap
rs = Sitemap(verbose=self.verbose,
allow_multifile=self.allow_multifile,
mapper=self.mapper)
if (self.verbose):
print "Reading sitemap(s) from %s ..." % (ref_sitemap)
ri = rs.read(ref_sitemap)
num_entries = len(ri)
print "Read reference sitemap with %d entries in %d sitemaps" % (
num_entries, rs.sitemaps_created)
if (self.verbose):
to_show = 100
override_str = ' (override with --max-sitemap-entries)'
if (self.max_sitemap_entries):
to_show = self.max_sitemap_entries
override_str = ''
if (num_entries > to_show):
print "Showing first %d entries sorted by URI%s..." % (
to_show, override_str)
n = 0
for r in ri.resource_uris():
print ri.resources[r]
n += 1
if (n >= to_show):
break
# 2. Set up base_path->base_uri mappings, get inventory from disk
disk_inventory = self.inventory
# 3. Calculate changeset
(num_same, updated, deleted, created) = ri.compare(disk_inventory)
changeset = Inventory()
changeset.capabilities = capabilities
changeset.add(disk_inventory.changeset(updated, changetype='updated'))
changeset.add(ri.changeset(deleted, changetype='deleted'))
changeset.add(disk_inventory.changeset(created, changetype='created'))
# 4. Write out changeset
s = Sitemap(verbose=self.verbose,
pretty_xml=True,
allow_multifile=self.allow_multifile,
mapper=self.mapper)
if (self.max_sitemap_entries is not None):
s.max_sitemap_entries = self.max_sitemap_entries
if (outfile is None):
print s.inventory_as_xml(changeset)
else:
s.write(changeset, basename=outfile)
self.write_dump_if_requested(changeset, dump)
def changeset_sitemap(self,outfile=None,ref_sitemap=None,capabilities=None,
dump=None):
# 1. Get and parse reference sitemap
rs = Sitemap(verbose=self.verbose, allow_multifile=self.allow_multifile,
mapper=self.mapper)
if (self.verbose):
print "Reading sitemap(s) from %s ..." % (ref_sitemap)
ri = rs.read(ref_sitemap)
num_entries = len(ri)
print "Read reference sitemap with %d entries in %d sitemaps" % (num_entries,rs.sitemaps_created)
if (self.verbose):
to_show = 100
override_str = ' (override with --max-sitemap-entries)'
if (self.max_sitemap_entries):
to_show = self.max_sitemap_entries
override_str = ''
if (num_entries>to_show):
print "Showing first %d entries sorted by URI%s..." % (to_show,override_str)
n=0
for r in ri.resource_uris():
print ri.resources[r]
n+=1
if ( n >= to_show ):
break
# 2. Set up base_path->base_uri mappings, get inventory from disk
disk_inventory = self.inventory
# 3. Calculate changeset
(num_same,updated,deleted,created)=ri.compare(disk_inventory)
changeset = Inventory()
changeset.capabilities = capabilities
changeset.add( disk_inventory.changeset( updated, changetype='updated' ) )
changeset.add( ri.changeset( deleted, changetype='deleted' ) )
changeset.add( disk_inventory.changeset( created, changetype='created' ) )
# 4. Write out changeset
s = Sitemap(verbose=self.verbose, pretty_xml=True, allow_multifile=self.allow_multifile,
mapper=self.mapper)
if (self.max_sitemap_entries is not None):
s.max_sitemap_entries = self.max_sitemap_entries
if (outfile is None):
print s.inventory_as_xml(changeset)
else:
s.write(changeset,basename=outfile)
self.write_dump_if_requested(changeset,dump)
def parse_sitemap(self):
s = Sitemap(allow_multifile=self.allow_multifile)
self.logger.info("Reading sitemap(s) from %s ..." % (self.sitemap))
i = s.read(self.sitemap)
num_entries = len(i)
self.logger.warning("Read sitemap with %d entries in %d sitemaps" % (num_entries, s.sitemaps_created))
if self.verbose:
to_show = 100
override_str = " (override with --max-sitemap-entries)"
if self.max_sitemap_entries:
to_show = self.max_sitemap_entries
override_str = ""
if num_entries > to_show:
print "Showing first %d entries sorted by URI%s..." % (to_show, override_str)
n = 0
for r in i:
print r
n += 1
if n >= to_show:
break
def parse_sitemap(self):
s=Sitemap(verbose=self.verbose, allow_multifile=self.allow_multifile)
if (self.verbose):
print "Reading sitemap(s) from %s ..." % (sitemap)
i = s.read(sitemap)
num_entries = len(i)
print "Read sitemap with %d entries in %d sitemaps" % (num_entries,s.sitemaps_created)
if (self.verbose):
to_show = 100
override_str = ' (override with --max-sitemap-entries)'
if (self.max_sitemap_entries):
to_show = self.max_sitemap_entries
override_str = ''
if (num_entries>to_show):
print "Showing first %d entries sorted by URI%s..." % (to_show,override_str)
n=0
for r in i.resource_uris():
print i.resources[r]
n+=1
if ( n >= to_show ):
break
def parse_sitemap(self):
s = Sitemap(allow_multifile=self.allow_multifile)
self.logger.info("Reading sitemap(s) from %s ..." % (self.sitemap))
i = s.read(self.sitemap)
num_entries = len(i)
self.logger.warning("Read sitemap with %d entries in %d sitemaps" %
(num_entries, s.sitemaps_created))
if (self.verbose):
to_show = 100
override_str = ' (override with --max-sitemap-entries)'
if (self.max_sitemap_entries):
to_show = self.max_sitemap_entries
override_str = ''
if (num_entries > to_show):
print "Showing first %d entries sorted by URI%s..." % (
to_show, override_str)
n = 0
for r in i:
print r
n += 1
if (n >= to_show):
break
def parse_sitemap(self):
s = Sitemap(verbose=self.verbose, allow_multifile=self.allow_multifile)
if (self.verbose):
print "Reading sitemap(s) from %s ..." % (sitemap)
i = s.read(sitemap)
num_entries = len(i)
print "Read sitemap with %d entries in %d sitemaps" % (
num_entries, s.sitemaps_created)
if (self.verbose):
to_show = 100
override_str = ' (override with --max-sitemap-entries)'
if (self.max_sitemap_entries):
to_show = self.max_sitemap_entries
override_str = ''
if (num_entries > to_show):
print "Showing first %d entries sorted by URI%s..." % (
to_show, override_str)
n = 0
for r in i.resource_uris():
print i.resources[r]
n += 1
if (n >= to_show):
break
def incremental(self, allow_deletion=False, changelist_uri=None):
"""Incremental synchronization"""
self.logger.debug("Starting incremental sync")
### 0. Sanity checks
if (len(self.mappings)<1):
raise ClientFatalError("No source to destination mapping specified")
# Get current config
inc_config_next=self.read_incremental_status(self.sitemap)
### 1. Get URI of changelist, from sitemap or explicit
if (inc_config_next is not None):
# We have config from last run for this site
changelist = inc_config_next
self.logger.info("ChangeList location from last incremental run %s" % (changelist))
elif (changelist_uri):
# Translate as necessary using maps
changelist = self.sitemap_changelist_uri(changelist_uri)
else:
# Get sitemap
try:
self.logger.info("Reading sitemap %s" % (self.sitemap))
src_sitemap = Sitemap(allow_multifile=self.allow_multifile, mapper=self.mapper)
src_resourcelist = src_sitemap.read(uri=self.sitemap, index_only=True)
self.logger.debug("Finished reading sitemap/sitemapindex")
except Exception as e:
raise ClientFatalError("Can't read source sitemap from %s (%s)" % (self.sitemap,str(e)))
# Extract changelist location
# FIXME - need to completely rework the way we handle/store capabilities
links = self.extract_links(src_resourcelist)
if ('current' not in links):
raise ClientFatalError("Failed to extract changelist location from sitemap %s" % (self.sitemap))
changelist = links['current']
### 2. Read changelist from source
ib = ResourceListBuilder(mapper=self.mapper)
try:
self.logger.info("Reading changelist %s" % (changelist))
src_sitemap = Sitemap(allow_multifile=self.allow_multifile, mapper=self.mapper)
src_changelist = src_sitemap.read(uri=changelist, changelist=True)
self.logger.debug("Finished reading changelist")
except Exception as e:
raise ClientFatalError("Can't read source changelist from %s (%s)" % (changelist,str(e)))
self.logger.info("Read source changelist, %d resources listed" % (len(src_changelist)))
#if (len(src_changelist)==0):
# raise ClientFatalError("Aborting as there are no resources to sync")
if (self.checksum and not src_changelist.has_md5()):
self.checksum=False
self.logger.info("Not calculating checksums on destination as not present in source resourcelist")
### 3. Check that sitemap has authority over URIs listed
# FIXME - What does authority mean for changelist? Here use both the
# changelist URI and, if we used it, the sitemap URI
uauth_cs = UrlAuthority(changelist)
if (not changelist_uri):
uauth_sm = UrlAuthority(self.sitemap)
for resource in src_changelist:
if (not uauth_cs.has_authority_over(resource.uri) and
(changelist_uri or not uauth_sm.has_authority_over(resource.uri))):
if (self.noauth):
#self.logger.info("ChangeList (%s) mentions resource at a location it does not have authority over (%s)" % (changelist,resource.uri))
pass
else:
raise ClientFatalError("Aborting as changelist (%s) mentions resource at a location it does not have authority over (%s), override with --noauth" % (changelist,resource.uri))
### 3. Apply changes
num_updated = 0
num_deleted = 0
num_created = 0
for resource in src_changelist:
uri = resource.uri
file = self.mapper.src_to_dst(uri)
if (resource.change == 'updated'):
self.logger.info("updated: %s -> %s" % (uri,file))
self.update_resource(resource,file,'updated')
num_updated+=1
elif (resource.change == 'created'):
self.logger.info("created: %s -> %s" % (uri,file))
self.update_resource(resource,file,'created')
num_created+=1
elif (resource.change == 'deleted'):
self.delete_resource(resource,file,allow_deletion)
num_deleted+=1
else:
raise ClientError("Unknown change type %s" % (resource.change) )
# 4. Report status and planned actions
status = "NO CHANGES"
if ((num_updated+num_deleted+num_created)>0):
status = " CHANGES "
self.logger.warning("Status: %s (updated=%d, deleted=%d, created=%d)" %\
(status,num_updated,num_deleted,num_created))
# 5. Store next link if available
if ((num_updated+num_deleted+num_created)>0):
links = self.extract_links(src_changelist)
if ('next' in links):
self.write_incremental_status(self.sitemap,links['next'])
self.logger.info("Written config with next incremental at %s" % (links['next']))
else:
self.logger.warning("Failed to extract next changelist location from changelist %s" % (changelist))
# 6. Done
self.logger.debug("Completed incremental sync")
def sync_or_audit(self, allow_deletion=False, audit_only=False):
action = "audit" if (audit_only) else "sync"
self.logger.debug("Starting " + action)
### 0. Sanity checks
if len(self.mappings) < 1:
raise ClientFatalError("No source to destination mapping specified")
### 1. Get inventories from both src and dst
# 1.a source inventory
ib = InventoryBuilder(mapper=self.mapper)
try:
self.logger.info("Reading sitemap %s" % (self.sitemap))
src_sitemap = Sitemap(allow_multifile=self.allow_multifile, mapper=self.mapper)
src_inventory = src_sitemap.read(uri=self.sitemap)
self.logger.debug("Finished reading sitemap")
except Exception as e:
raise ClientFatalError("Can't read source inventory from %s (%s)" % (self.sitemap, str(e)))
self.logger.info("Read source inventory, %d resources listed" % (len(src_inventory)))
if len(src_inventory) == 0:
raise ClientFatalError("Aborting as there are no resources to sync")
if self.checksum and not src_inventory.has_md5():
self.checksum = False
self.logger.info("Not calculating checksums on destination as not present in source inventory")
# 1.b destination inventory mapped back to source URIs
ib.do_md5 = self.checksum
dst_inventory = ib.from_disk()
### 2. Compare these inventorys respecting any comparison options
(same, updated, deleted, created) = dst_inventory.compare(src_inventory)
### 3. Report status and planned actions
status = " IN SYNC "
if len(updated) > 0 or len(deleted) > 0 or len(created) > 0:
status = "NOT IN SYNC"
self.logger.warning(
"Status: %s (same=%d, updated=%d, deleted=%d, created=%d)"
% (status, len(same), len(updated), len(deleted), len(created))
)
if audit_only:
self.logger.debug("Completed " + action)
return
### 4. Check that sitemap has authority over URIs listed
uauth = UrlAuthority(self.sitemap)
for resource in src_inventory:
if not uauth.has_authority_over(resource.uri):
if self.noauth:
self.logger.info(
"Sitemap (%s) mentions resource at a location it does not have authority over (%s)"
% (self.sitemap, resource.uri)
)
else:
raise ClientFatalError(
"Aborting as sitemap (%s) mentions resource at a location it does not have authority over (%s), override with --noauth"
% (self.sitemap, resource.uri)
)
### 5. Grab files to do sync
for resource in updated:
uri = resource.uri
file = self.mapper.src_to_dst(uri)
self.logger.info("updated: %s -> %s" % (uri, file))
self.update_resource(resource, file, "UPDATED")
for resource in created:
uri = resource.uri
file = self.mapper.src_to_dst(uri)
self.logger.info("created: %s -> %s" % (uri, file))
self.update_resource(resource, file, "CREATED")
for resource in deleted:
uri = resource.uri
if allow_deletion:
file = self.mapper.src_to_dst(uri)
if self.dryrun:
self.logger.info("dryrun: would delete %s -> %s" % (uri, file))
else:
os.unlink(file)
self.logger.info("deleted: %s -> %s" % (uri, file))
self.log_event(ResourceChange(resource=resource, changetype="DELETED"))
else:
self.logger.info("nodelete: would delete %s (--delete to enable)" % uri)
self.logger.debug("Completed " + action)
def incremental(self, allow_deletion=False, changeset_uri=None):
self.logger.debug("Starting incremental sync")
### 0. Sanity checks
if (len(self.mappings) < 1):
raise ClientFatalError(
"No source to destination mapping specified")
### 1. Get URI of changeset, from sitemap or explicit
if (changeset_uri):
# Translate as necessary using maps
changeset = self.sitemap_changeset_uri(changeset_uri)
else:
# Get sitemap
try:
self.logger.info("Reading sitemap %s" % (self.sitemap))
src_sitemap = Sitemap(allow_multifile=self.allow_multifile,
mapper=self.mapper)
src_inventory = src_sitemap.read(uri=self.sitemap,
index_only=True)
self.logger.debug("Finished reading sitemap/sitemapindex")
except Exception as e:
raise ClientFatalError(
"Can't read source sitemap from %s (%s)" %
(self.sitemap, str(e)))
# Extract changeset location
# FIXME - need to completely rework the way we handle/store capabilities
links = self.extract_links(src_inventory.capabilities)
if ('current' not in links):
raise ClientFatalError(
"Failed to extract changeset location from sitemap %s" %
(self.sitemap))
changeset = links['current']
### 2. Read changeset from source
ib = InventoryBuilder(mapper=self.mapper)
try:
self.logger.info("Reading changeset %s" % (changeset))
src_sitemap = Sitemap(allow_multifile=self.allow_multifile,
mapper=self.mapper)
src_changeset = src_sitemap.read(uri=changeset, changeset=True)
self.logger.debug("Finished reading changeset")
except Exception as e:
raise ClientFatalError("Can't read source changeset from %s (%s)" %
(changeset, str(e)))
self.logger.info("Read source changeset, %d resources listed" %
(len(src_changeset)))
if (len(src_changeset) == 0):
raise ClientFatalError(
"Aborting as there are no resources to sync")
if (self.checksum and not src_changeset.has_md5()):
self.checksum = False
self.logger.info(
"Not calculating checksums on destination as not present in source inventory"
)
### 3. Check that sitemap has authority over URIs listed
# FIXME - What does authority mean for changeset? Here use both the
# changeset URI and, if we used it, the sitemap URI
uauth_cs = UrlAuthority(changeset)
if (not changeset_uri):
uauth_sm = UrlAuthority(self.sitemap)
for resource in src_changeset:
if (not uauth_cs.has_authority_over(resource.uri)
and (changeset_uri
or not uauth_sm.has_authority_over(resource.uri))):
if (self.noauth):
self.logger.warning(
"Changeset (%s) mentions resource at a location it does not have authority over (%s)"
% (changeset, resource.uri))
else:
raise ClientFatalError(
"Aborting as changeset (%s) mentions resource at a location it does not have authority over (%s), override with --noauth"
% (changeset, resource.uri))
### 3. Apply changes
for resource in src_changeset:
uri = resource.uri
file = self.mapper.src_to_dst(uri)
if (resource.changetype == 'UPDATED'):
self.logger.info("updated: %s -> %s" % (uri, file))
self.update_resource(resource, file, 'UPDATED')
elif (resource.changetype == 'CREATED'):
self.logger.info("created: %s -> %s" % (uri, file))
self.update_resource(resource, file, 'CREATED')
elif (resource.changetype == 'DELETED'):
if (allow_deletion):
file = self.mapper.src_to_dst(uri)
if (self.dryrun):
self.logger.info("dryrun: would delete %s -> %s" %
(uri, file))
else:
os.unlink(file)
self.logger.info("deleted: %s -> %s" % (uri, file))
self.log_event(
ResourceChange(resource=resource,
changetype="DELETED"))
else:
self.logger.info(
"nodelete: would delete %s (--delete to enable)" % uri)
else:
raise ClientError("Unknown change type %s" %
(resource.changetype))
self.logger.debug("Completed incremental stuff")
def sync_or_audit(self, allow_deletion=False, audit_only=False):
action = ('audit' if (audit_only) else 'sync')
self.logger.debug("Starting " + action)
### 0. Sanity checks
if (len(self.mappings) < 1):
raise ClientFatalError(
"No source to destination mapping specified")
### 1. Get inventories from both src and dst
# 1.a source inventory
ib = InventoryBuilder(mapper=self.mapper)
try:
self.logger.info("Reading sitemap %s" % (self.sitemap))
src_sitemap = Sitemap(allow_multifile=self.allow_multifile,
mapper=self.mapper)
src_inventory = src_sitemap.read(uri=self.sitemap)
self.logger.debug("Finished reading sitemap")
except Exception as e:
raise ClientFatalError("Can't read source inventory from %s (%s)" %
(self.sitemap, str(e)))
self.logger.info("Read source inventory, %d resources listed" %
(len(src_inventory)))
if (len(src_inventory) == 0):
raise ClientFatalError(
"Aborting as there are no resources to sync")
if (self.checksum and not src_inventory.has_md5()):
self.checksum = False
self.logger.info(
"Not calculating checksums on destination as not present in source inventory"
)
# 1.b destination inventory mapped back to source URIs
ib.do_md5 = self.checksum
dst_inventory = ib.from_disk()
### 2. Compare these inventorys respecting any comparison options
(same, updated, deleted,
created) = dst_inventory.compare(src_inventory)
### 3. Report status and planned actions
status = " IN SYNC "
if (len(updated) > 0 or len(deleted) > 0 or len(created) > 0):
status = "NOT IN SYNC"
self.logger.warning("Status: %s (same=%d, updated=%d, deleted=%d, created=%d)" %\
(status,len(same),len(updated),len(deleted),len(created)))
if (audit_only):
self.logger.debug("Completed " + action)
return
### 4. Check that sitemap has authority over URIs listed
uauth = UrlAuthority(self.sitemap)
for resource in src_inventory:
if (not uauth.has_authority_over(resource.uri)):
if (self.noauth):
self.logger.info(
"Sitemap (%s) mentions resource at a location it does not have authority over (%s)"
% (self.sitemap, resource.uri))
else:
raise ClientFatalError(
"Aborting as sitemap (%s) mentions resource at a location it does not have authority over (%s), override with --noauth"
% (self.sitemap, resource.uri))
### 5. Grab files to do sync
for resource in updated:
uri = resource.uri
file = self.mapper.src_to_dst(uri)
self.logger.info("updated: %s -> %s" % (uri, file))
self.update_resource(resource, file, 'UPDATED')
for resource in created:
uri = resource.uri
file = self.mapper.src_to_dst(uri)
self.logger.info("created: %s -> %s" % (uri, file))
self.update_resource(resource, file, 'CREATED')
for resource in deleted:
uri = resource.uri
if (allow_deletion):
file = self.mapper.src_to_dst(uri)
if (self.dryrun):
self.logger.info("dryrun: would delete %s -> %s" %
(uri, file))
else:
os.unlink(file)
self.logger.info("deleted: %s -> %s" % (uri, file))
self.log_event(
ResourceChange(resource=resource,
changetype="DELETED"))
else:
self.logger.info(
"nodelete: would delete %s (--delete to enable)" % uri)
self.logger.debug("Completed " + action)
def read_example(self,file,changeset=False):
s = Sitemap()
r = s.read('resync/test/testdata/examples_from_spec/'+file)
self.assertEqual( s.changeset_read, changeset )
return r
def read_example(self, file, changeset=False):
s = Sitemap()
r = s.read('resync/test/testdata/examples_from_spec/' + file)
self.assertEqual(s.changeset_read, changeset)
return r
def incremental(self, allow_deletion=False, changeset_uri=None):
self.logger.debug("Starting incremental sync")
### 0. Sanity checks
if len(self.mappings) < 1:
raise ClientFatalError("No source to destination mapping specified")
### 1. Get URI of changeset, from sitemap or explicit
if changeset_uri:
# Translate as necessary using maps
changeset = self.sitemap_changeset_uri(changeset_uri)
else:
# Get sitemap
try:
self.logger.info("Reading sitemap %s" % (self.sitemap))
src_sitemap = Sitemap(allow_multifile=self.allow_multifile, mapper=self.mapper)
src_inventory = src_sitemap.read(uri=self.sitemap, index_only=True)
self.logger.debug("Finished reading sitemap/sitemapindex")
except Exception as e:
raise ClientFatalError("Can't read source sitemap from %s (%s)" % (self.sitemap, str(e)))
# Extract changeset location
# FIXME - need to completely rework the way we handle/store capabilities
links = self.extract_links(src_inventory.capabilities)
if "current" not in links:
raise ClientFatalError("Failed to extract changeset location from sitemap %s" % (self.sitemap))
changeset = links["current"]
### 2. Read changeset from source
ib = InventoryBuilder(mapper=self.mapper)
try:
self.logger.info("Reading changeset %s" % (changeset))
src_sitemap = Sitemap(allow_multifile=self.allow_multifile, mapper=self.mapper)
src_changeset = src_sitemap.read(uri=changeset, changeset=True)
self.logger.debug("Finished reading changeset")
except Exception as e:
raise ClientFatalError("Can't read source changeset from %s (%s)" % (changeset, str(e)))
self.logger.info("Read source changeset, %d resources listed" % (len(src_changeset)))
if len(src_changeset) == 0:
raise ClientFatalError("Aborting as there are no resources to sync")
if self.checksum and not src_changeset.has_md5():
self.checksum = False
self.logger.info("Not calculating checksums on destination as not present in source inventory")
### 3. Check that sitemap has authority over URIs listed
# FIXME - What does authority mean for changeset? Here use both the
# changeset URI and, if we used it, the sitemap URI
uauth_cs = UrlAuthority(changeset)
if not changeset_uri:
uauth_sm = UrlAuthority(self.sitemap)
for resource in src_changeset:
if not uauth_cs.has_authority_over(resource.uri) and (
changeset_uri or not uauth_sm.has_authority_over(resource.uri)
):
if self.noauth:
self.logger.warning(
"Changeset (%s) mentions resource at a location it does not have authority over (%s)"
% (changeset, resource.uri)
)
else:
raise ClientFatalError(
"Aborting as changeset (%s) mentions resource at a location it does not have authority over (%s), override with --noauth"
% (changeset, resource.uri)
)
### 3. Apply changes
for resource in src_changeset:
uri = resource.uri
file = self.mapper.src_to_dst(uri)
if resource.changetype == "UPDATED":
self.logger.info("updated: %s -> %s" % (uri, file))
self.update_resource(resource, file, "UPDATED")
elif resource.changetype == "CREATED":
self.logger.info("created: %s -> %s" % (uri, file))
self.update_resource(resource, file, "CREATED")
elif resource.changetype == "DELETED":
if allow_deletion:
file = self.mapper.src_to_dst(uri)
if self.dryrun:
self.logger.info("dryrun: would delete %s -> %s" % (uri, file))
else:
os.unlink(file)
self.logger.info("deleted: %s -> %s" % (uri, file))
self.log_event(ResourceChange(resource=resource, changetype="DELETED"))
else:
self.logger.info("nodelete: would delete %s (--delete to enable)" % uri)
else:
raise ClientError("Unknown change type %s" % (resource.changetype))
self.logger.debug("Completed incremental stuff")
def baseline_or_audit(self, allow_deletion=False, audit_only=False):
"""Baseline synchonization or audit
Both functions implemented in this routine because audit is a prerequisite
for a baseline sync.
"""
action = ( 'audit' if (audit_only) else 'baseline sync' )
self.logger.debug("Starting "+action)
### 0. Sanity checks
if (len(self.mappings)<1):
raise ClientFatalError("No source to destination mapping specified")
### 1. Get inventories from both src and dst
# 1.a source resourcelist
ib = ResourceListBuilder(mapper=self.mapper)
try:
self.logger.info("Reading sitemap %s" % (self.sitemap))
src_sitemap = Sitemap(allow_multifile=self.allow_multifile, mapper=self.mapper)
src_resourcelist = src_sitemap.read(uri=self.sitemap)
self.logger.debug("Finished reading sitemap")
except Exception as e:
raise ClientFatalError("Can't read source resourcelist from %s (%s)" % (self.sitemap,str(e)))
self.logger.info("Read source resourcelist, %d resources listed" % (len(src_resourcelist)))
if (len(src_resourcelist)==0):
raise ClientFatalError("Aborting as there are no resources to sync")
if (self.checksum and not src_resourcelist.has_md5()):
self.checksum=False
self.logger.info("Not calculating checksums on destination as not present in source resourcelist")
# 1.b destination resourcelist mapped back to source URIs
ib.do_md5=self.checksum
dst_resourcelist = ib.from_disk()
### 2. Compare these resourcelists respecting any comparison options
(same,updated,deleted,created)=dst_resourcelist.compare(src_resourcelist)
### 3. Report status and planned actions
status = " IN SYNC "
if (len(updated)>0 or len(deleted)>0 or len(created)>0):
status = "NOT IN SYNC"
self.logger.warning("Status: %s (same=%d, updated=%d, deleted=%d, created=%d)" %\
(status,len(same),len(updated),len(deleted),len(created)))
if (audit_only):
self.logger.debug("Completed "+action)
return
### 4. Check that sitemap has authority over URIs listed
uauth = UrlAuthority(self.sitemap)
for resource in src_resourcelist:
if (not uauth.has_authority_over(resource.uri)):
if (self.noauth):
#self.logger.info("Sitemap (%s) mentions resource at a location it does not have authority over (%s)" % (self.sitemap,resource.uri))
pass
else:
raise ClientFatalError("Aborting as sitemap (%s) mentions resource at a location it does not have authority over (%s), override with --noauth" % (self.sitemap,resource.uri))
### 5. Grab files to do sync
for resource in updated:
uri = resource.uri
file = self.mapper.src_to_dst(uri)
self.logger.info("updated: %s -> %s" % (uri,file))
self.update_resource(resource,file,'updated')
for resource in created:
uri = resource.uri
file = self.mapper.src_to_dst(uri)
self.logger.info("created: %s -> %s" % (uri,file))
self.update_resource(resource,file,'created')
for resource in deleted:
uri = resource.uri
file = self.mapper.src_to_dst(uri)
self.delete_resource(resource,file,allow_deletion)
### 6. For sync reset any incremental status for site
if (not audit_only):
links = self.extract_links(src_resourcelist)
if ('next' in links):
self.write_incremental_status(self.sitemap,links['next'])
self.logger.info("Written config with next incremental at %s" % (links['next']))
else:
self.write_incremental_status(self.sitemap)
self.logger.debug("Completed "+action)
