def __init__(self):
try:
self.consumer = KafkaConsumer("betweezered",
group_id="betweezered_consumer",
metadata_broker_list=["localhost:9092"])
logging.info("Initialized Apache Kafka connection.")
except:
logging.warning("Could not initialize Apache Kafka connection.")
def update_collection(name):
data = {}
# assuming POST
logging.debug(request.form)
# check for collection name
try:
c = models.Collection.objects.get(name=name)
except DoesNotExist:
logging.debug("collection does not exist, continuing")
return jsonify({'status':False})
c.search_terms = [term.strip() for term in request.form['search_terms'].split(",")]
c.minute_frequency = int(request.form['minute_frequency'])
c.save()
logging.debug("updated collection %s" % c.id)
# update cron job
job = mycron.find_comment('twitore_{name}'.format(name=name))
job = list(job)
logging.debug(job)
# update cron
'''
Consider improving the logic here
'''
if len(job) > 0:
if c.minute_frequency == 0:
utils.deleteCron(mycron, job[0], c)
elif c.minute_frequency >= 1 and c.minute_frequency <= 59:
utils.updateCron(mycron,job[0],c)
else:
logging.warning('frequency must be between 1-59 minutes. aborting.')
utils.setMsg(session,"collection could not be updated. frequency must be between 1-59 minutes. aborted.","msg_alert")
return redirect("/{prefix}/collection/{name}".format(prefix=localConfig.twitore_app_prefix,name=name))
# set cron if not there
else:
# set cron job
utils.setCron(mycron,c)
# set msg in session
utils.setMsg(session,"collection updated","msg_success")
return redirect("/{prefix}/collection/{name}".format(prefix=localConfig.twitore_app_prefix,name=name))
def processMessage(self, message):
try:
# retrieve payload and parse
payload = json.loads(message.value)
logging.info("tweet text: %s" % payload['text'])
# insert into MongoDB
try:
# insert tweet into Mongo
payload['id'] = str(payload['id']) # convert id to string
tweet = models.MongoTweet(**payload)
tweet.save()
logging.info("tweet inserted into db, id %s" % tweet.id)
except Exception, e:
logging.warning("could not insert into db. error: %s" % e)
except Exception, e:
logging.warning(e)
def collection(name):
# deal with message
msg = utils.msgHandle(session)
data = {}
c = models.Collection.objects.get(name=name)
data['collection'] = c
# stringify search terms
data['collection']['search_terms'] = ", ".join(data['collection']['search_terms'])
# get job cron
try:
job = mycron.find_comment('twitore_{name}'.format(name=name)).next()
logging.debug(job.command)
except:
logging.warning('could not find cron')
return render_template('collection.html', data=data, msg=msg)