def __synchronize__(self, destination):
config = Config()
checksum = config.boolean_prop(Config.key_use_checksum, True)
audit_only = config.boolean_prop(Config.key_audit_only, True)
allow_deletion = not audit_only
desclient = des.desclient.instance()
try:
desclient.set_mappings((self.uri, destination))
self.do_synchronize(desclient, allow_deletion, audit_only)
except ClientFatalError as err:
self.logger.warn("EXCEPTION while syncing %s" % self.uri, exc_info=True)
desclient.log_status(exception=err)
self.exceptions.append(err)
finally:
# A side effect (or a bug ;) is messing around with the
# class-level property Client.checksum. Make sure it is always set to initial value before the next
# source is processed.
desclient.checksum = checksum
def event_sitemap_received(self, uri, capability, text):
config = Config()
netloc = config.boolean_prop(Config.key_use_netloc, False)
baser_uri, local_path = DestinationMap().find_local_path(uri, netloc=netloc, infix=SITEMAP_FOLDER)
if local_path is not None:
os.makedirs(os.path.dirname(local_path), exist_ok=True)
with open(local_path, "w") as file:
file.write(text)
self.logger.debug("Saved %s '%s'" % (capability, local_path))
else:
self.logger.warn("Could not save %s. No local path for %s" % (capability, uri))
def process_source(self):
config = Config()
netloc = config.boolean_prop(Config.key_use_netloc, False)
base_uri, destination = DestinationMap().find_destination(self.uri, netloc=netloc, infix="resources")
if destination is None:
self.logger.debug("No destination for %s" % self.uri)
self.exceptions.append("No destination for %s" % self.uri)
des.reporter.instance().log_status(self.uri,
exception="No destination specified and use of net location prohibited.")
else:
self.__synchronize__(destination)
self.status = Status.processed_with_exceptions if self.has_exceptions() else Status.processed
def event_sitemap_received(self, uri, capability, text):
config = Config()
netloc = config.boolean_prop(Config.key_use_netloc, False)
baser_uri, local_path = DestinationMap().find_local_path(
uri, netloc=netloc, infix=SITEMAP_FOLDER)
if local_path is not None:
os.makedirs(os.path.dirname(local_path), exist_ok=True)
with open(local_path, "w") as file:
file.write(text)
self.logger.debug("Saved %s '%s'" % (capability, local_path))
else:
self.logger.warn("Could not save %s. No local path for %s" %
(capability, uri))
def instance():
"""
resync.Client is a somewhat heavy class. Desclient inherits and is adapted to be used during one run of
resyncing several sources. For convenience: grab the one instance from here.
:return: an instance of Desclient
"""
global _instance
logger = logging.getLogger(__name__)
if _instance is None:
config = Config()
# Parameters in the constructor of resync Client
checksum = config.boolean_prop(Config.key_use_checksum, True)
verbose = False
# Parameters in the method client.baseline_or_audit
audit_only = config.boolean_prop(Config.key_audit_only, True)
dryrun = audit_only
_instance = DesClient(checksum, verbose, dryrun)
logger.debug("Created a new %s [checksum=%s, verbose=%s, dryrun=%s]"
% ( _instance.__class__.__name__ , checksum, verbose, dryrun))
return _instance
def base_line(self, unzipdir):
"""
Synchronize the unzipped contents of a resource dump with the local resources
:param unzipdir: the directory of the unzipped packed contents.
:return:
"""
manifest_file_name = os.path.join(unzipdir, "manifest.xml")
try:
sitemap = Sitemap()
manifest_doc = sitemap.parse_xml(fh=manifest_file_name)
# the manifest_doc is a resync.resource_container.ResourceContainer
capability = manifest_doc.capability
assert capability == CAPA_RESOURCEDUMP_MANIFEST, "Capability is not %s but %s" % (
CAPA_RESOURCEDUMP_MANIFEST, capability)
self.status = Status.parsed
self.__inform_sitemap_received__(capability, manifest_file_name)
config = Config()
netloc = config.boolean_prop(Config.key_use_netloc, False)
base_uri, destination = DestinationMap().find_destination(
self.pack_uri, netloc=netloc)
assert destination is not None, "Found no destination folder in DestinationMap"
mapper = Mapper((base_uri, destination))
rlb = ResourceListBuilder(mapper=mapper)
dst_resource_list = rlb.from_disk()
# Compares on uri
same, updated, deleted, created = dst_resource_list.compare(
manifest_doc)
raise NotImplementedError("This class is not fully implemented.")
print(len(same), len(updated), len(deleted), len(created))
print("same")
for resource in same:
print(resource)
print("updated")
for resource in updated:
print(resource)
print("deleted")
for resource in deleted:
print(resource)
print("created")
for resource in created:
print(resource)
base_uri, local_path = DestinationMap().find_local_path(
resource.uri)
print(base_uri, local_path)
except AssertionError as err:
self.logger.debug("%s Error: %s" % (self.pack_uri, str(err)))
self.status = Status.parse_error
self.exceptions.append(err)
except SitemapParseError as err:
self.logger.debug("%s Unreadable source: %s" %
(self.source_uri, str(err)))
self.status = Status.parse_error
self.exceptions.append(err)
self.status = Status.processed_with_exceptions if self.has_exceptions(
) else Status.processed