예제 #1
0
파일: sync.py 프로젝트: EHRI/resydes
    def __synchronize__(self, destination):
        config = Config()
        checksum = config.boolean_prop(Config.key_use_checksum, True)
        audit_only = config.boolean_prop(Config.key_audit_only, True)
        allow_deletion = not audit_only

        desclient = des.desclient.instance()
        try:
            desclient.set_mappings((self.uri, destination))
            self.do_synchronize(desclient, allow_deletion, audit_only)
        except ClientFatalError as err:
            self.logger.warn("EXCEPTION while syncing %s" % self.uri, exc_info=True)
            desclient.log_status(exception=err)
            self.exceptions.append(err)
        finally:
            # A side effect (or a bug ;) is messing around with the
            # class-level property Client.checksum. Make sure it is always set to initial value before the next
            # source is processed.
            desclient.checksum = checksum
예제 #2
0
 def event_sitemap_received(self, uri, capability, text):
     config = Config()
     netloc = config.boolean_prop(Config.key_use_netloc, False)
     baser_uri, local_path = DestinationMap().find_local_path(uri, netloc=netloc, infix=SITEMAP_FOLDER)
     if local_path is not None:
         os.makedirs(os.path.dirname(local_path), exist_ok=True)
         with open(local_path, "w") as file:
             file.write(text)
         self.logger.debug("Saved %s '%s'" % (capability, local_path))
     else:
         self.logger.warn("Could not save %s. No local path for %s" % (capability, uri))
예제 #3
0
파일: sync.py 프로젝트: EHRI/resydes
    def process_source(self):
        config = Config()
        netloc = config.boolean_prop(Config.key_use_netloc, False)
        base_uri, destination = DestinationMap().find_destination(self.uri, netloc=netloc, infix="resources")
        if destination is None:
            self.logger.debug("No destination for %s" % self.uri)
            self.exceptions.append("No destination for %s" % self.uri)
            des.reporter.instance().log_status(self.uri,
                exception="No destination specified and use of net location prohibited.")
        else:
            self.__synchronize__(destination)

        self.status = Status.processed_with_exceptions if self.has_exceptions() else Status.processed
예제 #4
0
 def event_sitemap_received(self, uri, capability, text):
     config = Config()
     netloc = config.boolean_prop(Config.key_use_netloc, False)
     baser_uri, local_path = DestinationMap().find_local_path(
         uri, netloc=netloc, infix=SITEMAP_FOLDER)
     if local_path is not None:
         os.makedirs(os.path.dirname(local_path), exist_ok=True)
         with open(local_path, "w") as file:
             file.write(text)
         self.logger.debug("Saved %s '%s'" % (capability, local_path))
     else:
         self.logger.warn("Could not save %s. No local path for %s" %
                          (capability, uri))
예제 #5
0
파일: desclient.py 프로젝트: EHRI/resydes
def instance():
    """
    resync.Client is a somewhat heavy class. Desclient inherits and is adapted to be used during one run of
    resyncing several sources. For convenience: grab the one instance from here.
    :return: an instance of Desclient
    """
    global _instance
    logger = logging.getLogger(__name__)
    if _instance is None:
        config = Config()

        # Parameters in the constructor of resync Client
        checksum = config.boolean_prop(Config.key_use_checksum, True)
        verbose = False

        # Parameters in the method client.baseline_or_audit
        audit_only = config.boolean_prop(Config.key_audit_only, True)
        dryrun = audit_only

        _instance = DesClient(checksum, verbose, dryrun)
        logger.debug("Created a new %s [checksum=%s, verbose=%s, dryrun=%s]"
                         % ( _instance.__class__.__name__ , checksum, verbose, dryrun))

    return _instance
예제 #6
0
    def base_line(self, unzipdir):
        """
        Synchronize the unzipped contents of a resource dump with the local resources
        :param unzipdir: the directory of the unzipped packed contents.
        :return:
        """
        manifest_file_name = os.path.join(unzipdir, "manifest.xml")
        try:
            sitemap = Sitemap()
            manifest_doc = sitemap.parse_xml(fh=manifest_file_name)
            # the manifest_doc is a resync.resource_container.ResourceContainer
            capability = manifest_doc.capability
            assert capability == CAPA_RESOURCEDUMP_MANIFEST, "Capability is not %s but %s" % (
                CAPA_RESOURCEDUMP_MANIFEST, capability)
            self.status = Status.parsed
            self.__inform_sitemap_received__(capability, manifest_file_name)

            config = Config()
            netloc = config.boolean_prop(Config.key_use_netloc, False)
            base_uri, destination = DestinationMap().find_destination(
                self.pack_uri, netloc=netloc)
            assert destination is not None, "Found no destination folder in DestinationMap"
            mapper = Mapper((base_uri, destination))
            rlb = ResourceListBuilder(mapper=mapper)
            dst_resource_list = rlb.from_disk()
            # Compares on uri
            same, updated, deleted, created = dst_resource_list.compare(
                manifest_doc)

            raise NotImplementedError("This class is not fully implemented.")

            print(len(same), len(updated), len(deleted), len(created))

            print("same")
            for resource in same:
                print(resource)
            print("updated")
            for resource in updated:
                print(resource)
            print("deleted")
            for resource in deleted:
                print(resource)
            print("created")
            for resource in created:
                print(resource)
                base_uri, local_path = DestinationMap().find_local_path(
                    resource.uri)
                print(base_uri, local_path)

        except AssertionError as err:
            self.logger.debug("%s Error: %s" % (self.pack_uri, str(err)))
            self.status = Status.parse_error
            self.exceptions.append(err)
        except SitemapParseError as err:
            self.logger.debug("%s Unreadable source: %s" %
                              (self.source_uri, str(err)))
            self.status = Status.parse_error
            self.exceptions.append(err)

        self.status = Status.processed_with_exceptions if self.has_exceptions(
        ) else Status.processed