def update(identifiers, queue):
from ckanext.archiver import lib
for pkg_or_res, is_pkg, num_resources_for_pkg, pkg_for_res in \
_get_packages_and_resources_in_args(identifiers, queue):
if is_pkg:
package = pkg_or_res
log.info('Queuing dataset %s (%s resources) Q:%s', package.name,
num_resources_for_pkg, queue)
lib.create_archiver_package_task(package, queue)
sleep(0.1) # to try to avoid Redis getting overloaded
else:
resource = pkg_or_res
package = pkg_for_res
log.info('Queuing resource %s/%s', package.name, resource.id)
lib.create_archiver_resource_task(resource, queue)
sleep(0.05) # to try to avoid Redis getting overloaded
def update(self):
from ckanext.archiver import lib
for pkg_or_res, is_pkg, num_resources_for_pkg, pkg_for_res in \
self._get_packages_and_resources_in_args(self.args[1:]):
if is_pkg:
package = pkg_or_res
self.log.info('Queuing dataset %s (%s resources)',
package.name, num_resources_for_pkg)
lib.create_archiver_package_task(package, self.options.queue)
time.sleep(0.1) # to try to avoid Redis getting overloaded
else:
resource = pkg_or_res
package = pkg_for_res
self.log.info('Queuing resource %s/%s',
package.name, resource.id)
lib.create_archiver_resource_task(resource, self.options.queue)
time.sleep(0.05) # to try to avoid Redis getting overloaded
self.log.info('Completed queueing')
def update(self):
from ckanext.archiver import lib
for pkg_or_res, is_pkg, num_resources_for_pkg, pkg_for_res in \
self._get_packages_and_resources_in_args(self.args[1:]):
if is_pkg:
package = pkg_or_res
self.log.info('Queuing dataset %s (%s resources)',
package.name, num_resources_for_pkg)
lib.create_archiver_package_task(package, self.options.queue)
time.sleep(0.1) # to try to avoid Redis getting overloaded
else:
resource = pkg_or_res
package = pkg_for_res
self.log.info('Queuing resource %s/%s',
package.name, resource.id)
lib.create_archiver_resource_task(resource, self.options.queue)
time.sleep(0.05) # to try to avoid Redis getting overloaded
self.log.info('Completed queueing')
def update(self, dataset_spec, queue=None):
'''Archive all resources or just those belonging to a specific
package or group, if specified
'''
from ckanext.archiver import lib
for pkg_or_res, is_pkg, num_resources_for_pkg, pkg_for_res in \
self._get_packages_and_resources_in_args(dataset_spec):
if is_pkg:
package = pkg_or_res
log.info('Queuing dataset %s (%s resources)', package.name,
num_resources_for_pkg)
lib.create_archiver_package_task(package, self.queue)
time.sleep(0.1) # to try to avoid Redis getting overloaded
else:
resource = pkg_or_res
package = pkg_for_res
log.info('Queuing resource %s/%s', package.name, resource.id)
lib.create_archiver_resource_task(resource, self.queue)
time.sleep(0.05) # to try to avoid Redis getting overloaded
log.info('Completed queueing')
def update(self):
from ckan import model
from ckanext.archiver import lib
packages = []
resources = []
if len(self.args) > 1:
for arg in self.args[1:]:
# try arg as a group id/name
group = model.Group.get(arg)
if group:
if group.is_organization:
packages.extend(
model.Session.query(model.Package)
.filter_by(owner_org=group.id))
else:
packages.extend(group.packages(with_private=True))
if not self.options.queue:
self.options.queue = 'bulk'
continue
# try arg as a package id/name
pkg = model.Package.get(arg)
if pkg:
packages.append(pkg)
if not self.options.queue:
self.options.queue = 'priority'
continue
# try arg as a resource id
res = model.Resource.get(arg)
if res:
resources.append(res)
if not self.options.queue:
self.options.queue = 'priority'
continue
else:
self.log.error('Could not recognize as a group, package '
'or resource: %r', arg)
sys.exit(1)
else:
# all packages
pkgs = model.Session.query(model.Package)\
.filter_by(state='active')\
.order_by('name').all()
packages.extend(pkgs)
if not self.options.queue:
self.options.queue = 'bulk'
if packages:
self.log.info('Datasets to archive: %d', len(packages))
if resources:
self.log.info('Resources to archive: %d', len(resources))
if not (packages or resources):
self.log.error('No datasets or resources to process')
sys.exit(1)
self.log.info('Queue: %s', self.options.queue)
for package in packages:
if p.toolkit.check_ckan_version(max_version='2.2.99'):
# earlier CKANs had ResourceGroup
pkg_resources = \
[res for res in
itertools.chain.from_iterable(
(rg.resources_all
for rg in package.resource_groups_all)
)
if res.state == 'active']
else:
pkg_resources = \
[res for res in package.resources_all
if res.state == 'active']
self.log.info('Queuing dataset %s (%s resources)',
package.name, len(pkg_resources))
lib.create_archiver_package_task(package, self.options.queue)
time.sleep(0.1) # to try to avoid Redis getting overloaded
for resource in resources:
if p.toolkit.check_ckan_version(max_version='2.2.99'):
package = resource.resource_group.package
else:
package = resource.package
self.log.info('Queuing resource %s/%s', package.name, resource.id)
lib.create_archiver_resource_task(resource, self.options.queue)
time.sleep(0.05) # to try to avoid Redis getting overloaded
self.log.info('Completed queueing')
def update(self):
from ckan import model
from ckanext.archiver import lib
packages = []
resources = []
if len(self.args) > 1:
for arg in self.args[1:]:
# try arg as a group id/name
group = model.Group.get(arg)
if group:
if group.is_organization:
packages.extend(
model.Session.query(
model.Package).filter_by(owner_org=group.id))
else:
packages.extend(group.packages(with_private=True))
if not self.options.queue:
self.options.queue = 'bulk'
continue
# try arg as a package id/name
pkg = model.Package.get(arg)
if pkg:
packages.append(pkg)
if not self.options.queue:
self.options.queue = 'priority'
continue
# try arg as a resource id
res = model.Resource.get(arg)
if res:
resources.append(res)
if not self.options.queue:
self.options.queue = 'priority'
continue
else:
self.log.error(
'Could not recognize as a group, package '
'or resource: %r', arg)
sys.exit(1)
else:
# all packages
pkgs = model.Session.query(model.Package)\
.filter_by(state='active')\
.order_by('name').all()
packages.extend(pkgs)
if not self.options.queue:
self.options.queue = 'bulk'
if packages:
self.log.info('Datasets to archive: %d', len(packages))
if resources:
self.log.info('Resources to archive: %d', len(resources))
if not (packages or resources):
self.log.error('No datasets or resources to process')
sys.exit(1)
self.log.info('Queue: %s', self.options.queue)
for package in packages:
if hasattr(model, 'ResourceGroup'):
# earlier CKANs had ResourceGroup
pkg_resources = \
[res for res in
itertools.chain.from_iterable(
(rg.resources_all
for rg in package.resource_groups_all)
)
if res.state == 'active']
else:
pkg_resources = \
[res for res in package.resources_all
if res.state == 'active']
self.log.info('Queuing dataset %s (%s resources)', package.name,
len(pkg_resources))
lib.create_archiver_package_task(package, self.options.queue)
time.sleep(0.1) # to try to avoid Redis getting overloaded
for resource in resources:
package = resource.resource_group.package
self.log.info('Queuing resource %s/%s', package.name, resource.id)
lib.create_archiver_resource_task(resource, self.options.queue)
time.sleep(0.05) # to try to avoid Redis getting overloaded
self.log.info('Completed queueing')