def take_action(self, args): dataset = get_dataset(self.app.crawler, args.dataset) datasource = unicode(args.datasource) if datasource in dataset: raise ValueError('Datasource "%s" exists in "%s" dataset') kwargs = dict((key, getattr(args, key))\ for key in args.__dict__.keys()\ if not key in self.OMIT_DATASOURCE_ARGS) dataset[datasource] = args.datasource_class(dataset._path, **kwargs) dataset.save() self.app.log.info('Created "%s" datasource in "%s" dataset')
def take_action(self, args): if isinstance(args.handle, basestring): args.handle = get_handler_obj(value, args.handle) root_handler = logging.getLogger('') handlers = root_handler.handlers root_handler.handlers = [] dataset = get_dataset(self.app.crawler, args.dataset) urls = non_fnmatchers(dataset) for datasource, urls in map_datasources(urls, dataset).items(): for item in self.crawl(urls, datasource): args.handle(item) root_handler.handlers = handlers
def take_action(self, args): dataset = get_dataset(self.app.crawler, args.dataset) datasource = unicode(args.datasource) del dataset[datasource] dataset.save()
def take_action(self, args): dataset = get_dataset(self.app.crawler, args.dataset) headers = ('Tree',) content = (''.join(dataset.pformat(indent=' ')),) return headers, content
def take_action(self, args): dataset = get_dataset(self.app.crawler, args.dataset) headers = ("Tree",) content = ("".join(dataset.pformat(indent=" ")),) return headers, content