def setUp(self):
os.environ["redis_log_keyname_key"] = "briefcam"
os.environ["total_job_enqueued_count_redis_name_key"] = "enqueue"
os.environ["total_job_dequeued_count_redis_name_key"] = "dequeue"
self.dirname = os.path.dirname(os.path.realpath(__file__))
self.create_test_docker_container()
self.redisInterface = RedisInterface()
class TestRedisInterface(unittest.TestCase):
def setUp(self):
os.environ["redis_log_keyname_key"] = "briefcam"
os.environ["total_job_enqueued_count_redis_name_key"] = "enqueue"
os.environ["total_job_dequeued_count_redis_name_key"] = "dequeue"
self.dirname = os.path.dirname(os.path.realpath(__file__))
self.create_test_docker_container()
self.redisInterface = RedisInterface()
def test_redis_interface(self):
self.assertEqual(self.redisInterface.get_current_enqueue_count(), -1)
self.assertEqual(self.redisInterface.get_current_dequeue_count(), -1)
self.redisInterface.increment_enqueue_count()
self.assertEqual(
self.redisInterface.get_current_enqueue_count().decode('utf8'),
'1')
self.redisInterface.increment_dequeue_count()
self.assertEqual(
self.redisInterface.get_current_enqueue_count().decode('utf8'),
'1')
def create_test_docker_container(self):
completedProcess = subprocess.run([
"docker-compose", "-f", "{}/docker-compose_redis.yml".format(
self.dirname), "up", "-d"
],
stdout=subprocess.PIPE)
self.assertIsNotNone(completedProcess)
self.assertIsNotNone(completedProcess.stdout)
# time.sleep(120)
def delete_test_docker_container(self):
completedProcess = subprocess.run([
"docker-compose", "-f", "{}/docker-compose_redis.yml".format(
self.dirname), "down"
],
stdout=subprocess.PIPE)
self.assertIsNotNone(completedProcess)
self.assertIsNotNone(completedProcess.stdout)
def tearDown(self):
self.delete_test_docker_container()
from redis_interface import RedisInterface
from nltk.chunk import RegexpParser
from operator import itemgetter
from time import sleep
from tagged_article import TaggedArticle
def sanitizeTags(taggedList):
sanitizedList = []
for key, value in taggedList:
if not value:
value = 'NNP'
sanitizedList.append((key, value))
return sanitizedList
# Open the redis interface
redisInterface = RedisInterface()
# Prepare the chunker
chunker = RegexpParser(r'''
Nouns:
{<JJ.*>*<NN.*>*}
''');
# Print status
print 'Analyzer ONLINE'
# Run the wait-execute loop
articleNumber = 0
while True:
while not redisInterface.hasArticleData(articleNumber, 'article_data'):
for taggedString in paraWithTags.strings:
paraString += removeApostrophe(taggedString.string)
return paraString
def titleFromArticleSoup(soup):
titleDiv = soup.find(class_ = 'story-heading')
if not titleDiv:
titleDiv = soup.find(class_ = 'entry-title')
return unicode(removeApostrophe(titleDiv.string))
# Set up the tokenizer and the tagger
tokenizer = RegexpTokenizer(r'\w+')
tagger = UnigramTagger(treebank.tagged_sents())
# Open up a redis connection
redisInterface = RedisInterface()
# Print status
print 'Reader ONLINE'
# Run the wait-execute loop
while True:
while not redisInterface.hasPending():
sleep(1)
page = redisInterface.popPending()
print 'Reading ' + page + ' STARTED'
# Read the html page
with open(page, 'r') as htmlPage:
