def setUp(self):
SolrTestCase.setUp(self)
SitemapTestCase.setUp(self)
args = Namespace(tika=None, solr=None,
slacktoken=None, slackchannel=None)
args.config = BASIC_CONFIG
self.config = deepcopy(get_config(args))
self.config.url_field = 'url'
def setUp(self):
CrawlerTestCase.setUp(self)
args = Namespace(tika=None,
solr=None,
slacktoken=None,
slackchannel=None)
args.config = BASIC_CONFIG
self.config = deepcopy(get_config(args))
def test_get_config_loads_config_module_and_returns_config_instance(self):
options = Namespace(tika=None,
solr=None,
slacktoken=None,
slackchannel=None)
options.config = BASIC_CONFIG
config = get_config(options)
self.assertIsInstance(config, Config)
def main():
options = parse_args()
config = get_config(options)
tempdir = tempfile.mkdtemp(prefix='ftw.crawler_')
log.debug(u"Using temporary directory {}".format(tempdir))
try:
crawl_and_index(tempdir, config, options)
finally:
shutil.rmtree(tempdir)
def setUp(self):
SolrTestCase.setUp(self)
SitemapTestCase.setUp(self)
args = Namespace(tika=None,
solr=None,
slacktoken=None,
slackchannel=None)
args.config = BASIC_CONFIG
self.config = deepcopy(get_config(args))
self.config.url_field = 'url'
def test_get_config_sets_arguments_from_command_line(self):
options = Namespace(tika='http://tika',
solr='http://solr',
slacktoken='token',
slackchannel='#channel')
options.config = BASIC_CONFIG
config = get_config(options)
self.assertEquals('http://tika', config.tika)
self.assertEquals('http://solr', config.solr)
self.assertEquals('token', config.slacktoken)
self.assertEquals('#channel', config.slackchannel)
def setUp(self):
CrawlerTestCase.setUp(self)
args = Namespace(tika=None, solr=None,
slacktoken=None, slackchannel=None)
args.config = BASIC_CONFIG
self.config = deepcopy(get_config(args))