def update_test(identifiers, queue): from ckanext.archiver import tasks # Prevent it loading config again tasks.load_config = lambda x: None for pkg_or_res, is_pkg, num_resources_for_pkg, pkg_for_res in \ _get_packages_and_resources_in_args(identifiers): if is_pkg: package = pkg_or_res log.info('Archiving dataset %s (%s resources)', package.name, num_resources_for_pkg) tasks._update_package(package.id, queue, log) else: resource = pkg_or_res package = pkg_for_res log.info('Queuing resource %s/%s', package.name, resource.id) tasks._update_resource(resource.id, queue, log)
def update_test(self, dataset_spec, queue=None): from ckanext.archiver import tasks log = logging.getLogger('ckanext.archiver') for pkg_or_res, is_pkg, num_resources_for_pkg, pkg_for_res in \ self._get_packages_and_resources_in_args(dataset_spec): if is_pkg: package = pkg_or_res log.info('Archiving dataset %s (%s resources)', package.name, num_resources_for_pkg) tasks._update_package(package.id, self.queue, log) else: resource = pkg_or_res package = pkg_for_res log.info('Archiving resource %s/%s', package.name, resource.id) tasks._update_resource(resource.id, self.queue, log) log.info('Completed test update')
def update_test(self): from ckanext.archiver import tasks # Prevent it loading config again tasks.load_config = lambda x: None log = logging.getLogger('ckanext.archiver') for pkg_or_res, is_pkg, num_resources_for_pkg, pkg_for_res in \ self._get_packages_and_resources_in_args(self.args[1:]): if is_pkg: package = pkg_or_res self.log.info('Archiving dataset %s (%s resources)', package.name, num_resources_for_pkg) tasks._update_package(package.id, self.options.queue, log) else: resource = pkg_or_res package = pkg_for_res self.log.info('Queuing resource %s/%s', package.name, resource.id) tasks._update_resource(resource.id, self.options.queue, log) self.log.info('Completed test update')
cl = headers.get('content-length') if resource.get('mimetype') != ct: resource_changed = True resource['mimetype'] = ct # this is to store the size in case there is an error, but the real size # check is done after dowloading the data file, with its real length if cl is not None and (resource.get('size') != cl): resource_changed = True resource['size'] = cl # make sure resource content-length does not exceed our maximum if cl and int(cl) >= max_content_length: if resource_changed: tasks._update_resource(context, resource, log) # record fact that resource is too large to archive raise tasks.DownloadError("Content-length {0} exceeds maximum allowed" "value {1}".format(cl, max_content_length)) # check that resource is a data file if not (resource_format in data_formats or ct.lower().strip('"') in data_formats): if resource_changed: tasks._update_resource(context, resource, log) raise tasks.DownloadError("Of content type {0}, not " "downloading".format(ct)) # get the resource and archive it # TODO: remove the Accept-Encoding limitation after upgrading # archiver and requests if not res:
cl = headers.get('content-length') if resource.get('mimetype') != ct: resource_changed = True resource['mimetype'] = ct # this is to store the size in case there is an error, but the real size # check is done after dowloading the data file, with its real length if cl is not None and (resource.get('size') != cl): resource_changed = True resource['size'] = cl # make sure resource content-length does not exceed our maximum if cl and int(cl) >= max_content_length: if resource_changed: tasks._update_resource(context, resource, log) # record fact that resource is too large to archive raise tasks.DownloadError("Content-length {0} exceeds maximum allowed" "value {1}".format(cl, max_content_length)) # check that resource is a data file if not (resource_format in data_formats or ct.lower().strip('"') in data_formats): if resource_changed: tasks._update_resource(context, resource, log) raise tasks.DownloadError("Of content type {0}, not " "downloading".format(ct)) # get the resource and archive it # TODO: remove the Accept-Encoding limitation after upgrading # archiver and requests