Esempio n. 1
0
def update(identifiers, queue):
    from ckanext.archiver import lib
    for pkg_or_res, is_pkg, num_resources_for_pkg, pkg_for_res in \
            _get_packages_and_resources_in_args(identifiers, queue):
        if is_pkg:
            package = pkg_or_res
            log.info('Queuing dataset %s (%s resources) Q:%s', package.name,
                     num_resources_for_pkg, queue)
            lib.create_archiver_package_task(package, queue)
            sleep(0.1)  # to try to avoid Redis getting overloaded
        else:
            resource = pkg_or_res
            package = pkg_for_res
            log.info('Queuing resource %s/%s', package.name, resource.id)
            lib.create_archiver_resource_task(resource, queue)
            sleep(0.05)  # to try to avoid Redis getting overloaded
Esempio n. 2
0
 def update(self):
     from ckanext.archiver import lib
     for pkg_or_res, is_pkg, num_resources_for_pkg, pkg_for_res in \
             self._get_packages_and_resources_in_args(self.args[1:]):
         if is_pkg:
             package = pkg_or_res
             self.log.info('Queuing dataset %s (%s resources)',
                           package.name, num_resources_for_pkg)
             lib.create_archiver_package_task(package, self.options.queue)
             time.sleep(0.1)  # to try to avoid Redis getting overloaded
         else:
             resource = pkg_or_res
             package = pkg_for_res
             self.log.info('Queuing resource %s/%s',
                           package.name, resource.id)
             lib.create_archiver_resource_task(resource, self.options.queue)
             time.sleep(0.05)  # to try to avoid Redis getting overloaded
     self.log.info('Completed queueing')
Esempio n. 3
0
 def update(self):
     from ckanext.archiver import lib
     for pkg_or_res, is_pkg, num_resources_for_pkg, pkg_for_res in \
             self._get_packages_and_resources_in_args(self.args[1:]):
         if is_pkg:
             package = pkg_or_res
             self.log.info('Queuing dataset %s (%s resources)',
                           package.name, num_resources_for_pkg)
             lib.create_archiver_package_task(package, self.options.queue)
             time.sleep(0.1)  # to try to avoid Redis getting overloaded
         else:
             resource = pkg_or_res
             package = pkg_for_res
             self.log.info('Queuing resource %s/%s',
                           package.name, resource.id)
             lib.create_archiver_resource_task(resource, self.options.queue)
             time.sleep(0.05)  # to try to avoid Redis getting overloaded
     self.log.info('Completed queueing')
Esempio n. 4
0
 def update(self, dataset_spec, queue=None):
     '''Archive all resources or just those belonging to a specific
     package or group, if specified
     '''
     from ckanext.archiver import lib
     for pkg_or_res, is_pkg, num_resources_for_pkg, pkg_for_res in \
             self._get_packages_and_resources_in_args(dataset_spec):
         if is_pkg:
             package = pkg_or_res
             log.info('Queuing dataset %s (%s resources)', package.name,
                      num_resources_for_pkg)
             lib.create_archiver_package_task(package, self.queue)
             time.sleep(0.1)  # to try to avoid Redis getting overloaded
         else:
             resource = pkg_or_res
             package = pkg_for_res
             log.info('Queuing resource %s/%s', package.name, resource.id)
             lib.create_archiver_resource_task(resource, self.queue)
             time.sleep(0.05)  # to try to avoid Redis getting overloaded
     log.info('Completed queueing')
Esempio n. 5
0
    def update(self):
        from ckan import model
        from ckanext.archiver import lib
        packages = []
        resources = []
        if len(self.args) > 1:
            for arg in self.args[1:]:
                # try arg as a group id/name
                group = model.Group.get(arg)
                if group:
                    if group.is_organization:
                        packages.extend(
                            model.Session.query(model.Package)
                                 .filter_by(owner_org=group.id))
                    else:
                        packages.extend(group.packages(with_private=True))
                    if not self.options.queue:
                        self.options.queue = 'bulk'
                    continue
                # try arg as a package id/name
                pkg = model.Package.get(arg)
                if pkg:
                    packages.append(pkg)
                    if not self.options.queue:
                        self.options.queue = 'priority'
                    continue
                # try arg as a resource id
                res = model.Resource.get(arg)
                if res:
                    resources.append(res)
                    if not self.options.queue:
                        self.options.queue = 'priority'
                    continue
                else:
                    self.log.error('Could not recognize as a group, package '
                                   'or resource: %r', arg)
                    sys.exit(1)
        else:
            # all packages
            pkgs = model.Session.query(model.Package)\
                        .filter_by(state='active')\
                        .order_by('name').all()
            packages.extend(pkgs)
            if not self.options.queue:
                self.options.queue = 'bulk'

        if packages:
            self.log.info('Datasets to archive: %d', len(packages))
        if resources:
            self.log.info('Resources to archive: %d', len(resources))
        if not (packages or resources):
            self.log.error('No datasets or resources to process')
            sys.exit(1)

        self.log.info('Queue: %s', self.options.queue)
        for package in packages:
            if p.toolkit.check_ckan_version(max_version='2.2.99'):
                # earlier CKANs had ResourceGroup
                pkg_resources = \
                    [res for res in
                        itertools.chain.from_iterable(
                            (rg.resources_all
                             for rg in package.resource_groups_all)
                        )
                     if res.state == 'active']
            else:
                pkg_resources = \
                    [res for res in package.resources_all
                     if res.state == 'active']
            self.log.info('Queuing dataset %s (%s resources)',
                          package.name, len(pkg_resources))
            lib.create_archiver_package_task(package, self.options.queue)
            time.sleep(0.1)  # to try to avoid Redis getting overloaded

        for resource in resources:
            if p.toolkit.check_ckan_version(max_version='2.2.99'):
                package = resource.resource_group.package
            else:
                package = resource.package
            self.log.info('Queuing resource %s/%s', package.name, resource.id)
            lib.create_archiver_resource_task(resource, self.options.queue)
            time.sleep(0.05)  # to try to avoid Redis getting overloaded

        self.log.info('Completed queueing')
Esempio n. 6
0
    def update(self):
        from ckan import model
        from ckanext.archiver import lib
        packages = []
        resources = []
        if len(self.args) > 1:
            for arg in self.args[1:]:
                # try arg as a group id/name
                group = model.Group.get(arg)
                if group:
                    if group.is_organization:
                        packages.extend(
                            model.Session.query(
                                model.Package).filter_by(owner_org=group.id))
                    else:
                        packages.extend(group.packages(with_private=True))
                    if not self.options.queue:
                        self.options.queue = 'bulk'
                    continue
                # try arg as a package id/name
                pkg = model.Package.get(arg)
                if pkg:
                    packages.append(pkg)
                    if not self.options.queue:
                        self.options.queue = 'priority'
                    continue
                # try arg as a resource id
                res = model.Resource.get(arg)
                if res:
                    resources.append(res)
                    if not self.options.queue:
                        self.options.queue = 'priority'
                    continue
                else:
                    self.log.error(
                        'Could not recognize as a group, package '
                        'or resource: %r', arg)
                    sys.exit(1)
        else:
            # all packages
            pkgs = model.Session.query(model.Package)\
                        .filter_by(state='active')\
                        .order_by('name').all()
            packages.extend(pkgs)
            if not self.options.queue:
                self.options.queue = 'bulk'

        if packages:
            self.log.info('Datasets to archive: %d', len(packages))
        if resources:
            self.log.info('Resources to archive: %d', len(resources))
        if not (packages or resources):
            self.log.error('No datasets or resources to process')
            sys.exit(1)

        self.log.info('Queue: %s', self.options.queue)
        for package in packages:
            if hasattr(model, 'ResourceGroup'):
                # earlier CKANs had ResourceGroup
                pkg_resources = \
                    [res for res in
                        itertools.chain.from_iterable(
                            (rg.resources_all
                             for rg in package.resource_groups_all)
                        )
                     if res.state == 'active']
            else:
                pkg_resources = \
                    [res for res in package.resources_all
                     if res.state == 'active']
            self.log.info('Queuing dataset %s (%s resources)', package.name,
                          len(pkg_resources))
            lib.create_archiver_package_task(package, self.options.queue)
            time.sleep(0.1)  # to try to avoid Redis getting overloaded

        for resource in resources:
            package = resource.resource_group.package
            self.log.info('Queuing resource %s/%s', package.name, resource.id)
            lib.create_archiver_resource_task(resource, self.options.queue)
            time.sleep(0.05)  # to try to avoid Redis getting overloaded

        self.log.info('Completed queueing')