def main(conf_file):
import logging.config
logging.config.fileConfig(conf_file, disable_existing_loggers=False)
logger = logging.getLogger(__name__)
logger.info('Starting')
cluster = Cluster('127.0.0.1')
session = cluster.connect('iote2e')
logger.info(session)
cluster.close()
def get(ip, keyspace):
connection = pika.BlockingConnection(
pika.ConnectionParameters(os.getenv("RABBIT_HOST"),
heartbeat=5,
socket_timeout=1,
blocked_connection_timeout=1))
channel = connection.channel()
channel.queue_declare(queue='articles')
article = SimpleStatement("""
INSERT INTO article (url, timeuuid, category, topimage, title, publication, summary, articletext, html, assets)
VALUES (%(url)s, now(), %(category)s, %(topimage)s, %(title)s, %(publication)s, %(summary)s, %(articletext)s, %(html)s, %(assets)s)
""",
consistency_level=ConsistencyLevel.ONE)
article_category = SimpleStatement("""
INSERT INTO article_category (url, timeuuid, category, topimage, title, publication, summary, articletext, html, assets)
VALUES (%(url)s, now(), %(category)s, %(topimage)s, %(title)s, %(publication)s, %(summary)s, %(articletext)s, %(html)s, %(assets)s)
""",
consistency_level=ConsistencyLevel.ONE)
article_publication = SimpleStatement(
"""
INSERT INTO article_publication (url, timeuuid, category, topimage, title, publication, summary, articletext, html, assets)
VALUES (%(url)s, now(), %(category)s, %(topimage)s, %(title)s, %(publication)s, %(summary)s, %(articletext)s, %(html)s, %(assets)s)
""",
consistency_level=ConsistencyLevel.ONE)
method_frame, header_frame, body = channel.basic_get('articles')
if method_frame:
pathuuid = str(uuid.uuid4())
try:
channel.basic_ack(method_frame.delivery_tag)
cluster = Cluster([ip])
session = cluster.connect()
session.set_keyspace(keyspace)
try:
parsed = parsearticle.parsearticle(body, pathuuid)
except:
shutil.rmtree(pathuuid)
articlequeue.get()
print('failed')
exit(1)
try:
for asset in parsed['assets']:
thing = chunkcass.chunkandinsertimage(
session=session,
filepath=asset['imgpath'],
imgname=asset['imgname'],
imgurl=asset['imgurl'],
content_type=asset['content_type'])
shutil.rmtree(pathuuid)
session.execute(
article,
dict(url=str(parsed['articleurl']),
title=parsed['title'],
publication=parsed['publication'],
category=parsed['category'],
topimage=str(parsed['assets'][0]['imgurl']),
summary=parsed['summary'],
articletext=parsed['articletext'],
html=parsed['html'],
assets=str(parsed['assets'])))
session.execute(
article_category,
dict(url=str(parsed['articleurl']),
title=parsed['title'],
publication=parsed['publication'],
category=parsed['category'],
topimage=str(parsed['assets'][0]['imgurl']),
summary=parsed['summary'],
articletext=parsed['articletext'],
html=parsed['html'],
assets=str(parsed['assets'])))
session.execute(
article_publication,
dict(url=str(parsed['articleurl']),
title=parsed['title'],
publication=parsed['publication'],
category=parsed['category'],
topimage=str(parsed['assets'][0]['imgurl']),
summary=parsed['summary'],
articletext=parsed['articletext'],
html=parsed['html'],
assets=str(parsed['assets'])))
session.close()
cluster.close()
except:
session.close()
cluster.close()
shutil.rmtree(pathuuid)
articlequeue.get()
print('failed')
exit(1)
except:
session.close()
cluster.close()
shutil.rmtree(pathuuid)
articlequeue.get()
print('failed')
exit(1)
else:
session.close()
cluster.close()
connection.close()
sleep(60)
# print('No message returned')
try:
articlequeue.get()
except:
pass