def setUp(self): config.load(configfile) # Re fetch and/or parse the test feeds spider.spiderPlanet() global testfeeds testfeeds = glob.glob(spider_workdir+"/*") try: os.makedirs(workdir) except: self.tearDown() os.makedirs(workdir)
def test_spiderThreads(self): config.load(configfile.replace('config', 'threaded')) _PORT = config.parser.getint('Planet', 'test_port') log = [] from SimpleHTTPServer import SimpleHTTPRequestHandler class TestRequestHandler(SimpleHTTPRequestHandler): def log_message(self, format_, *args): log.append(args) from threading import Thread class TestServerThread(Thread): def __init__(self): self.ready = 0 self.done = 0 Thread.__init__(self) def run(self): from BaseHTTPServer import HTTPServer httpd_ = HTTPServer(('', _PORT), TestRequestHandler) self.ready = 1 while not self.done: httpd_.handle_request() httpd = TestServerThread() httpd.start() while not httpd.ready: time.sleep(0.1) try: spiderPlanet() finally: httpd.done = 1 import urllib urllib.urlopen('http://127.0.0.1:%d/' % _PORT).read() status = [int(rec[1]) for rec in log if str(rec[0]).startswith('GET ')] status.sort() self.assertEqual([200, 200, 200, 200, 404], status) self.verify_spiderPlanet()
def test_index_spider(self): import test_spider config.load(test_spider.configfile) index = idindex.create() self.assertEqual(0, len(index)) index.close() from planet.spider import spiderPlanet try: spiderPlanet() index = idindex.open() self.assertEqual(12, len(index)) self.assertEqual('tag:planet.intertwingly.net,2006:testfeed1', index['planet.intertwingly.net,2006,testfeed1,1']) self.assertEqual('http://intertwingly.net/code/venus/tests/data/spider/testfeed3.rss', index['planet.intertwingly.net,2006,testfeed3,1']) index.close() finally: import os, shutil shutil.rmtree(test_spider.workdir) os.removedirs(os.path.split(test_spider.workdir)[0])
def test_spiderThreads(self): config.load(configfile.replace('config','threaded')) _PORT = config.parser.getint('Planet','test_port') log = [] from SimpleHTTPServer import SimpleHTTPRequestHandler class TestRequestHandler(SimpleHTTPRequestHandler): def log_message(self, format, *args): log.append(args) from threading import Thread class TestServerThread(Thread): def __init__(self): self.ready = 0 self.done = 0 Thread.__init__(self) def run(self): from BaseHTTPServer import HTTPServer httpd = HTTPServer(('',_PORT), TestRequestHandler) self.ready = 1 while not self.done: httpd.handle_request() httpd = TestServerThread() httpd.start() while not httpd.ready: time.sleep(0.1) try: spiderPlanet() finally: httpd.done = 1 import urllib urllib.urlopen('http://127.0.0.1:%d/' % _PORT).read() status = [int(rec[1]) for rec in log if str(rec[0]).startswith('GET ')] status.sort() self.assertEqual([200,200,200,200,404], status) self.verify_spiderPlanet()
def test_index_spider(self): import test_spider config.load(test_spider.configfile) index = idindex.create() self.assertEqual(0, len(index)) index.close() from planet.spider import spiderPlanet try: spiderPlanet() index = idindex.open() self.assertEqual(12, len(index)) self.assertEqual('tag:planet.intertwingly.net,2006:testfeed1', index['planet.intertwingly.net,2006,testfeed1,1']) self.assertEqual( 'http://intertwingly.net/code/venus/tests/data/spider/testfeed3.rss', index['planet.intertwingly.net,2006,testfeed3,1']) index.close() finally: shutil.rmtree(test_spider.workdir) os.removedirs(os.path.split(test_spider.workdir)[0])
parser.set('Planet','cache_directory',work) parser.set('Planet','output_dir',output) parser.set('Planet','filter_directories',filters) if hide_planet_ns: parser.set('Planet','template_files','themes/common/atom.xml.xslt') else: parser.set('Planet','template_files','tests/data/reconstitute.xslt') for name, value in zip(sys.argv[2::2],sys.argv[3::2]): parser.set(sys.argv[1], name.lstrip('-'), value) from planet import config config.parser = parser from planet import spider spider.spiderPlanet(only_if_new=False) import feedparser for source in glob.glob(os.path.join(work, 'sources/*')): feed = feedparser.parse(source).feed if feed.has_key('title'): config.parser.set('Planet','name',feed.title_detail.value) if feed.has_key('link'): config.parser.set('Planet','link',feed.link) if feed.has_key('author_detail'): if feed.author_detail.has_key('name'): config.parser.set('Planet','owner_name',feed.author_detail.name) if feed.author_detail.has_key('email'): config.parser.set('Planet','owner_email',feed.author_detail.email) from planet import splice
def test_spiderPlanet(self): config.load(configfile) spiderPlanet() self.verify_spiderPlanet()
print >>sys.stderr, "Unknown option:", arg sys.exit(1) else: config_file.append(arg) from planet import config config.load(config_file or 'config.ini') if verbose: import planet planet.getLogger('DEBUG',config.log_format()) if not offline: from planet import spider try: spider.spiderPlanet(only_if_new=only_if_new) except Exception, e: print e from planet import splice doc = splice.splice() if debug_splice: from planet import logger logger.info('writing debug.atom') debug=open('debug.atom','w') try: from lxml import etree from StringIO import StringIO tree = etree.tostring(etree.parse(StringIO(doc.toxml()))) debug.write(etree.tostring(tree, pretty_print=True))
a major change in the contract between stages """ import shutil, os, sys # move up a directory sys.path.insert(0, os.path.split(sys.path[0])[0]) os.chdir(sys.path[0]) # copy spider output to splice input import planet from planet import spider, config planet.getLogger('CRITICAL',None) config.load('tests/data/spider/config.ini') spider.spiderPlanet() if os.path.exists('tests/data/splice/cache'): shutil.rmtree('tests/data/splice/cache') shutil.move('tests/work/spider/cache', 'tests/data/splice/cache') source=open('tests/data/spider/config.ini') dest1=open('tests/data/splice/config.ini', 'w') dest1.write(source.read().replace('/work/spider/', '/data/splice/')) dest1.close() source.seek(0) dest2=open('tests/work/apply_config.ini', 'w') dest2.write(source.read().replace('[Planet]', '''[Planet] output_theme = asf output_dir = tests/work/apply''')) dest2.close()
config_file.append(arg) import locale locale.setlocale(locale.LC_ALL, "") from planet import config config.load(config_file or 'config.ini') if verbose: import planet planet.getLogger('DEBUG', config.log_format()) if not offline: from planet import spider try: spider.spiderPlanet(only_if_new=only_if_new) except Exception, e: print e from planet import splice doc = splice.splice() if debug_splice: from planet import logger logger.info('writing debug.atom') debug = open('debug.atom', 'w') try: from lxml import etree from StringIO import StringIO tree = etree.tostring(etree.parse(StringIO(doc.toxml()))) debug.write(etree.tostring(tree, pretty_print=True))
""" import shutil, os, sys # move up a directory sys.path.insert(0, os.path.split(sys.path[0])[0]) os.chdir(sys.path[0]) # copy spider output to splice input import planet from planet import spider, config planet.getLogger('CRITICAL', None) config.load('tests/data/spider/config.ini') spider.spiderPlanet() if os.path.exists('tests/data/splice/cache'): shutil.rmtree('tests/data/splice/cache') shutil.move('tests/work/spider/cache', 'tests/data/splice/cache') source = open('tests/data/spider/config.ini') dest1 = open('tests/data/splice/config.ini', 'w') dest1.write(source.read().replace('/work/spider/', '/data/splice/')) dest1.close() source.seek(0) dest2 = open('tests/work/apply_config.ini', 'w') dest2.write(source.read().replace( '[Planet]', '''[Planet] output_theme = asf output_dir = tests/work/apply'''))
parser.set('Planet', 'cache_directory', work) parser.set('Planet', 'output_dir', output) parser.set('Planet', 'filter_directories', filters) if hide_planet_ns: parser.set('Planet', 'template_files', 'themes/common/atom.xml.xslt') else: parser.set('Planet', 'template_files', 'tests/data/reconstitute.xslt') for name, value in zip(sys.argv[2::2], sys.argv[3::2]): parser.set(sys.argv[1], name.lstrip('-'), value) from planet import config config.parser = parser from planet import spider spider.spiderPlanet(only_if_new=False) import feedparser for source in glob.glob(os.path.join(work, 'sources/*')): feed = feedparser.parse(source).feed if feed.has_key('title'): config.parser.set('Planet', 'name', feed.title_detail.value) if feed.has_key('link'): config.parser.set('Planet', 'link', feed.link) if feed.has_key('author_detail'): if feed.author_detail.has_key('name'): config.parser.set('Planet', 'owner_name', feed.author_detail.name) if feed.author_detail.has_key('email'): config.parser.set('Planet', 'owner_email', feed.author_detail.email)