def setUp(self):
config.load(configfile)
# Re fetch and/or parse the test feeds
spider.spiderPlanet()
global testfeeds
testfeeds = glob.glob(spider_workdir+"/*")
try:
os.makedirs(workdir)
except:
self.tearDown()
os.makedirs(workdir)
def test_spiderThreads(self):
config.load(configfile.replace('config', 'threaded'))
_PORT = config.parser.getint('Planet', 'test_port')
log = []
from SimpleHTTPServer import SimpleHTTPRequestHandler
class TestRequestHandler(SimpleHTTPRequestHandler):
def log_message(self, format_, *args):
log.append(args)
from threading import Thread
class TestServerThread(Thread):
def __init__(self):
self.ready = 0
self.done = 0
Thread.__init__(self)
def run(self):
from BaseHTTPServer import HTTPServer
httpd_ = HTTPServer(('', _PORT), TestRequestHandler)
self.ready = 1
while not self.done:
httpd_.handle_request()
httpd = TestServerThread()
httpd.start()
while not httpd.ready:
time.sleep(0.1)
try:
spiderPlanet()
finally:
httpd.done = 1
import urllib
urllib.urlopen('http://127.0.0.1:%d/' % _PORT).read()
status = [int(rec[1]) for rec in log if str(rec[0]).startswith('GET ')]
status.sort()
self.assertEqual([200, 200, 200, 200, 404], status)
self.verify_spiderPlanet()
def test_index_spider(self):
import test_spider
config.load(test_spider.configfile)
index = idindex.create()
self.assertEqual(0, len(index))
index.close()
from planet.spider import spiderPlanet
try:
spiderPlanet()
index = idindex.open()
self.assertEqual(12, len(index))
self.assertEqual('tag:planet.intertwingly.net,2006:testfeed1', index['planet.intertwingly.net,2006,testfeed1,1'])
self.assertEqual('http://intertwingly.net/code/venus/tests/data/spider/testfeed3.rss', index['planet.intertwingly.net,2006,testfeed3,1'])
index.close()
finally:
import os, shutil
shutil.rmtree(test_spider.workdir)
os.removedirs(os.path.split(test_spider.workdir)[0])
def test_spiderThreads(self):
config.load(configfile.replace('config','threaded'))
_PORT = config.parser.getint('Planet','test_port')
log = []
from SimpleHTTPServer import SimpleHTTPRequestHandler
class TestRequestHandler(SimpleHTTPRequestHandler):
def log_message(self, format, *args):
log.append(args)
from threading import Thread
class TestServerThread(Thread):
def __init__(self):
self.ready = 0
self.done = 0
Thread.__init__(self)
def run(self):
from BaseHTTPServer import HTTPServer
httpd = HTTPServer(('',_PORT), TestRequestHandler)
self.ready = 1
while not self.done:
httpd.handle_request()
httpd = TestServerThread()
httpd.start()
while not httpd.ready:
time.sleep(0.1)
try:
spiderPlanet()
finally:
httpd.done = 1
import urllib
urllib.urlopen('http://127.0.0.1:%d/' % _PORT).read()
status = [int(rec[1]) for rec in log if str(rec[0]).startswith('GET ')]
status.sort()
self.assertEqual([200,200,200,200,404], status)
self.verify_spiderPlanet()
def test_index_spider(self):
import test_spider
config.load(test_spider.configfile)
index = idindex.create()
self.assertEqual(0, len(index))
index.close()
from planet.spider import spiderPlanet
try:
spiderPlanet()
index = idindex.open()
self.assertEqual(12, len(index))
self.assertEqual('tag:planet.intertwingly.net,2006:testfeed1',
index['planet.intertwingly.net,2006,testfeed1,1'])
self.assertEqual(
'http://intertwingly.net/code/venus/tests/data/spider/testfeed3.rss',
index['planet.intertwingly.net,2006,testfeed3,1'])
index.close()
finally:
shutil.rmtree(test_spider.workdir)
os.removedirs(os.path.split(test_spider.workdir)[0])
parser.set('Planet','cache_directory',work)
parser.set('Planet','output_dir',output)
parser.set('Planet','filter_directories',filters)
if hide_planet_ns:
parser.set('Planet','template_files','themes/common/atom.xml.xslt')
else:
parser.set('Planet','template_files','tests/data/reconstitute.xslt')
for name, value in zip(sys.argv[2::2],sys.argv[3::2]):
parser.set(sys.argv[1], name.lstrip('-'), value)
from planet import config
config.parser = parser
from planet import spider
spider.spiderPlanet(only_if_new=False)
import feedparser
for source in glob.glob(os.path.join(work, 'sources/*')):
feed = feedparser.parse(source).feed
if feed.has_key('title'):
config.parser.set('Planet','name',feed.title_detail.value)
if feed.has_key('link'):
config.parser.set('Planet','link',feed.link)
if feed.has_key('author_detail'):
if feed.author_detail.has_key('name'):
config.parser.set('Planet','owner_name',feed.author_detail.name)
if feed.author_detail.has_key('email'):
config.parser.set('Planet','owner_email',feed.author_detail.email)
from planet import splice
def test_spiderPlanet(self):
config.load(configfile)
spiderPlanet()
self.verify_spiderPlanet()
print >>sys.stderr, "Unknown option:", arg
sys.exit(1)
else:
config_file.append(arg)
from planet import config
config.load(config_file or 'config.ini')
if verbose:
import planet
planet.getLogger('DEBUG',config.log_format())
if not offline:
from planet import spider
try:
spider.spiderPlanet(only_if_new=only_if_new)
except Exception, e:
print e
from planet import splice
doc = splice.splice()
if debug_splice:
from planet import logger
logger.info('writing debug.atom')
debug=open('debug.atom','w')
try:
from lxml import etree
from StringIO import StringIO
tree = etree.tostring(etree.parse(StringIO(doc.toxml())))
debug.write(etree.tostring(tree, pretty_print=True))
a major change in the contract between stages
"""
import shutil, os, sys
# move up a directory
sys.path.insert(0, os.path.split(sys.path[0])[0])
os.chdir(sys.path[0])
# copy spider output to splice input
import planet
from planet import spider, config
planet.getLogger('CRITICAL',None)
config.load('tests/data/spider/config.ini')
spider.spiderPlanet()
if os.path.exists('tests/data/splice/cache'):
shutil.rmtree('tests/data/splice/cache')
shutil.move('tests/work/spider/cache', 'tests/data/splice/cache')
source=open('tests/data/spider/config.ini')
dest1=open('tests/data/splice/config.ini', 'w')
dest1.write(source.read().replace('/work/spider/', '/data/splice/'))
dest1.close()
source.seek(0)
dest2=open('tests/work/apply_config.ini', 'w')
dest2.write(source.read().replace('[Planet]', '''[Planet]
output_theme = asf
output_dir = tests/work/apply'''))
dest2.close()
config_file.append(arg)
import locale
locale.setlocale(locale.LC_ALL, "")
from planet import config
config.load(config_file or 'config.ini')
if verbose:
import planet
planet.getLogger('DEBUG', config.log_format())
if not offline:
from planet import spider
try:
spider.spiderPlanet(only_if_new=only_if_new)
except Exception, e:
print e
from planet import splice
doc = splice.splice()
if debug_splice:
from planet import logger
logger.info('writing debug.atom')
debug = open('debug.atom', 'w')
try:
from lxml import etree
from StringIO import StringIO
tree = etree.tostring(etree.parse(StringIO(doc.toxml())))
debug.write(etree.tostring(tree, pretty_print=True))
"""
import shutil, os, sys
# move up a directory
sys.path.insert(0, os.path.split(sys.path[0])[0])
os.chdir(sys.path[0])
# copy spider output to splice input
import planet
from planet import spider, config
planet.getLogger('CRITICAL', None)
config.load('tests/data/spider/config.ini')
spider.spiderPlanet()
if os.path.exists('tests/data/splice/cache'):
shutil.rmtree('tests/data/splice/cache')
shutil.move('tests/work/spider/cache', 'tests/data/splice/cache')
source = open('tests/data/spider/config.ini')
dest1 = open('tests/data/splice/config.ini', 'w')
dest1.write(source.read().replace('/work/spider/', '/data/splice/'))
dest1.close()
source.seek(0)
dest2 = open('tests/work/apply_config.ini', 'w')
dest2.write(source.read().replace(
'[Planet]', '''[Planet]
output_theme = asf
output_dir = tests/work/apply'''))
parser.set('Planet', 'cache_directory', work)
parser.set('Planet', 'output_dir', output)
parser.set('Planet', 'filter_directories', filters)
if hide_planet_ns:
parser.set('Planet', 'template_files', 'themes/common/atom.xml.xslt')
else:
parser.set('Planet', 'template_files', 'tests/data/reconstitute.xslt')
for name, value in zip(sys.argv[2::2], sys.argv[3::2]):
parser.set(sys.argv[1], name.lstrip('-'), value)
from planet import config
config.parser = parser
from planet import spider
spider.spiderPlanet(only_if_new=False)
import feedparser
for source in glob.glob(os.path.join(work, 'sources/*')):
feed = feedparser.parse(source).feed
if feed.has_key('title'):
config.parser.set('Planet', 'name', feed.title_detail.value)
if feed.has_key('link'):
config.parser.set('Planet', 'link', feed.link)
if feed.has_key('author_detail'):
if feed.author_detail.has_key('name'):
config.parser.set('Planet', 'owner_name',
feed.author_detail.name)
if feed.author_detail.has_key('email'):
config.parser.set('Planet', 'owner_email',
feed.author_detail.email)
