def take_action(self, args):
dataset = get_dataset(self.app.crawler, args.dataset)
datasource = unicode(args.datasource)
if datasource in dataset:
raise ValueError('Datasource "%s" exists in "%s" dataset')
kwargs = dict((key, getattr(args, key))\
for key in args.__dict__.keys()\
if not key in self.OMIT_DATASOURCE_ARGS)
dataset[datasource] = args.datasource_class(dataset._path, **kwargs)
dataset.save()
self.app.log.info('Created "%s" datasource in "%s" dataset')
def take_action(self, args):
dataset = get_dataset(self.app.crawler, args.dataset)
datasource = unicode(args.datasource)
if datasource in dataset:
raise ValueError('Datasource "%s" exists in "%s" dataset')
kwargs = dict((key, getattr(args, key))\
for key in args.__dict__.keys()\
if not key in self.OMIT_DATASOURCE_ARGS)
dataset[datasource] = args.datasource_class(dataset._path, **kwargs)
dataset.save()
self.app.log.info('Created "%s" datasource in "%s" dataset')
def take_action(self, args):
if isinstance(args.handle, basestring):
args.handle = get_handler_obj(value, args.handle)
root_handler = logging.getLogger('')
handlers = root_handler.handlers
root_handler.handlers = []
dataset = get_dataset(self.app.crawler, args.dataset)
urls = non_fnmatchers(dataset)
for datasource, urls in map_datasources(urls, dataset).items():
for item in self.crawl(urls, datasource):
args.handle(item)
root_handler.handlers = handlers
def take_action(self, args):
if isinstance(args.handle, basestring):
args.handle = get_handler_obj(value, args.handle)
root_handler = logging.getLogger('')
handlers = root_handler.handlers
root_handler.handlers = []
dataset = get_dataset(self.app.crawler, args.dataset)
urls = non_fnmatchers(dataset)
for datasource, urls in map_datasources(urls, dataset).items():
for item in self.crawl(urls, datasource):
args.handle(item)
root_handler.handlers = handlers
def take_action(self, args):
dataset = get_dataset(self.app.crawler, args.dataset)
datasource = unicode(args.datasource)
del dataset[datasource]
dataset.save()
def take_action(self, args):
dataset = get_dataset(self.app.crawler, args.dataset)
datasource = unicode(args.datasource)
del dataset[datasource]
dataset.save()
def take_action(self, args):
dataset = get_dataset(self.app.crawler, args.dataset)
headers = ('Tree',)
content = (''.join(dataset.pformat(indent=' ')),)
return headers, content
def take_action(self, args):
dataset = get_dataset(self.app.crawler, args.dataset)
headers = ("Tree",)
content = ("".join(dataset.pformat(indent=" ")),)
return headers, content
