def update_test(identifiers, queue):
from ckanext.archiver import tasks
# Prevent it loading config again
tasks.load_config = lambda x: None
for pkg_or_res, is_pkg, num_resources_for_pkg, pkg_for_res in \
_get_packages_and_resources_in_args(identifiers):
if is_pkg:
package = pkg_or_res
log.info('Archiving dataset %s (%s resources)', package.name,
num_resources_for_pkg)
tasks._update_package(package.id, queue, log)
else:
resource = pkg_or_res
package = pkg_for_res
log.info('Queuing resource %s/%s', package.name, resource.id)
tasks._update_resource(resource.id, queue, log)
def update_test(self, dataset_spec, queue=None):
from ckanext.archiver import tasks
log = logging.getLogger('ckanext.archiver')
for pkg_or_res, is_pkg, num_resources_for_pkg, pkg_for_res in \
self._get_packages_and_resources_in_args(dataset_spec):
if is_pkg:
package = pkg_or_res
log.info('Archiving dataset %s (%s resources)', package.name,
num_resources_for_pkg)
tasks._update_package(package.id, self.queue, log)
else:
resource = pkg_or_res
package = pkg_for_res
log.info('Archiving resource %s/%s', package.name, resource.id)
tasks._update_resource(resource.id, self.queue, log)
log.info('Completed test update')
def update_test(self):
from ckanext.archiver import tasks
# Prevent it loading config again
tasks.load_config = lambda x: None
log = logging.getLogger('ckanext.archiver')
for pkg_or_res, is_pkg, num_resources_for_pkg, pkg_for_res in \
self._get_packages_and_resources_in_args(self.args[1:]):
if is_pkg:
package = pkg_or_res
self.log.info('Archiving dataset %s (%s resources)',
package.name, num_resources_for_pkg)
tasks._update_package(package.id, self.options.queue, log)
else:
resource = pkg_or_res
package = pkg_for_res
self.log.info('Queuing resource %s/%s',
package.name, resource.id)
tasks._update_resource(resource.id, self.options.queue, log)
self.log.info('Completed test update')
def update_test(self):
from ckanext.archiver import tasks
# Prevent it loading config again
tasks.load_config = lambda x: None
log = logging.getLogger('ckanext.archiver')
for pkg_or_res, is_pkg, num_resources_for_pkg, pkg_for_res in \
self._get_packages_and_resources_in_args(self.args[1:]):
if is_pkg:
package = pkg_or_res
self.log.info('Archiving dataset %s (%s resources)',
package.name, num_resources_for_pkg)
tasks._update_package(package.id, self.options.queue, log)
else:
resource = pkg_or_res
package = pkg_for_res
self.log.info('Queuing resource %s/%s',
package.name, resource.id)
tasks._update_resource(resource.id, self.options.queue, log)
self.log.info('Completed test update')
cl = headers.get('content-length')
if resource.get('mimetype') != ct:
resource_changed = True
resource['mimetype'] = ct
# this is to store the size in case there is an error, but the real size
# check is done after dowloading the data file, with its real length
if cl is not None and (resource.get('size') != cl):
resource_changed = True
resource['size'] = cl
# make sure resource content-length does not exceed our maximum
if cl and int(cl) >= max_content_length:
if resource_changed:
tasks._update_resource(context, resource, log)
# record fact that resource is too large to archive
raise tasks.DownloadError("Content-length {0} exceeds maximum allowed"
"value {1}".format(cl, max_content_length))
# check that resource is a data file
if not (resource_format in data_formats or ct.lower().strip('"') in data_formats):
if resource_changed:
tasks._update_resource(context, resource, log)
raise tasks.DownloadError("Of content type {0}, not "
"downloading".format(ct))
# get the resource and archive it
# TODO: remove the Accept-Encoding limitation after upgrading
# archiver and requests
if not res:
cl = headers.get('content-length')
if resource.get('mimetype') != ct:
resource_changed = True
resource['mimetype'] = ct
# this is to store the size in case there is an error, but the real size
# check is done after dowloading the data file, with its real length
if cl is not None and (resource.get('size') != cl):
resource_changed = True
resource['size'] = cl
# make sure resource content-length does not exceed our maximum
if cl and int(cl) >= max_content_length:
if resource_changed:
tasks._update_resource(context, resource, log)
# record fact that resource is too large to archive
raise tasks.DownloadError("Content-length {0} exceeds maximum allowed"
"value {1}".format(cl, max_content_length))
# check that resource is a data file
if not (resource_format in data_formats
or ct.lower().strip('"') in data_formats):
if resource_changed:
tasks._update_resource(context, resource, log)
raise tasks.DownloadError("Of content type {0}, not "
"downloading".format(ct))
# get the resource and archive it
# TODO: remove the Accept-Encoding limitation after upgrading
# archiver and requests
