def test_saveurl(self):
conn = HarvestManUrlConnector()
url = random.choice(urls)
res = conn.save_url(HarvestManUrl(url))
if conn.get_error().number == 0:
assert (res == DOWNLOAD_YES_OK)
if os.path.isfile('index.html'):
os.remove('index.html')
else:
print 'Error in fetching data, skipping tests...'
def test_saveurl(self):
conn = HarvestManUrlConnector()
url = random.choice(urls)
res = conn.save_url(HarvestManUrl(url))
if conn.get_error().number==0:
assert(res==DOWNLOAD_YES_OK)
if os.path.isfile('index.html'):
os.remove('index.html')
else:
print 'Error in fetching data, skipping tests...'
def test_urltofile(self):
objects.config.showprogress = False
conn = HarvestManUrlConnector()
url = random.choice(urls)
res = conn.url_to_file(HarvestManUrl(url))
if conn.get_error().number==0:
assert(res==URL_DOWNLOAD_OK)
if os.path.isfile('index.html'):
os.remove('index.html')
else:
print 'Error in fetching data, skipping tests...'
def test_urltofile(self):
objects.config.showprogress = False
conn = HarvestManUrlConnector()
url = random.choice(urls)
res = conn.url_to_file(HarvestManUrl(url))
if conn.get_error().number == 0:
assert (res == URL_DOWNLOAD_OK)
if os.path.isfile('index.html'):
os.remove('index.html')
else:
print 'Error in fetching data, skipping tests...'
def open(self, url):
return HarvestManUrlConnector().robot_urlopen(url)
def test_connect(self):
conn = HarvestManUrlConnector()
url = random.choice(urls)
res = conn.connect(HarvestManUrl(url))
error = conn.get_error()
if error.number == 0:
assert (res == CONNECT_YES_DOWNLOADED)
assert (conn.get_content_length() > 0)
content_type = conn.get_content_type()
assert (content_type == 'text/html')
fo = conn.get_fileobj()
assert (fo != None)
assert (fo.get_data() == '')
# Since default is flushing to file, the file
# object should not be None
assert (fo.get_tmpfile() != None)
else:
print 'Error in fetching data, skipping tests...'
# Now set connector to in-mem mode and test again
objects.config.datamode = CONNECTOR_DATA_MODE_INMEM
conn = HarvestManUrlConnector()
url = random.choice(urls)
res = conn.connect(HarvestManUrl(url))
# There could be an error...
error = conn.get_error()
if error.number == 0:
assert (res == CONNECT_YES_DOWNLOADED)
assert (conn.get_content_length() > 0)
content_type = conn.get_content_type()
assert (content_type == 'text/html')
fo = conn.get_fileobj()
assert (fo != None)
assert (fo.get_data() != '')
assert (fo.get_tmpfile() == None)
else:
print 'Error in fetching data, skipping tests...'
def test_connect(self):
conn = HarvestManUrlConnector()
url = random.choice(urls)
res = conn.connect(HarvestManUrl(url))
error = conn.get_error()
if error.number==0:
assert(res == CONNECT_YES_DOWNLOADED)
assert(conn.get_content_length()>0)
content_type = conn.get_content_type()
assert(content_type == 'text/html')
fo = conn.get_fileobj()
assert(fo != None)
assert(fo.get_data() == '')
# Since default is flushing to file, the file
# object should not be None
assert(fo.get_tmpfile() != None)
else:
print 'Error in fetching data, skipping tests...'
# Now set connector to in-mem mode and test again
objects.config.datamode = CONNECTOR_DATA_MODE_INMEM
conn = HarvestManUrlConnector()
url = random.choice(urls)
res = conn.connect(HarvestManUrl(url))
# There could be an error...
error = conn.get_error()
if error.number==0:
assert(res == CONNECT_YES_DOWNLOADED)
assert(conn.get_content_length()>0)
content_type = conn.get_content_type()
assert(content_type == 'text/html')
fo = conn.get_fileobj()
assert(fo != None)
assert(fo.get_data() != '')
assert(fo.get_tmpfile() == None)
else:
print 'Error in fetching data, skipping tests...'