예제 #1
0
def delete_collection(name):

	# retrieve collection mongo record
	logging.debug("attempting to delete %s" % name)
	try:
		c = models.Collection.objects.get(name=name)        
	except DoesNotExist:
		logging.debug("could not find collection to delete")
		return jsonify({'status':False})

	# delete collection
	models.Collection.delete(c)
	'''
	Consider removing archived tweets from filesystem?
	Consider removing archived tweets from monogodb?
	'''	

	# remove from crontab
	job = list(mycron.find_comment('twitore_%s' % name))
	logging.debug(job)
	if len(job) == 1:
		utils.deleteCron(mycron, job[0], c)		
	else:
		logging.debug('could not find cron to delete')

	# set msg in session		
	utils.setMsg(session,"collection deleted","msg_success")

	# if all goes well...	
	return redirect("/{prefix}/collections".format(prefix=localConfig.twitore_app_prefix))
예제 #2
0
def collection_tweets(name,page=1):

	c = models.Collection.objects.get(name=name)

	data = {
		"collection":c
	}

	pt = models.MongoTweet.objects.filter(twitore_collection=name).order_by('-id').paginate(page=page, per_page=25)

	for tweet in pt.items:
		logging.debug("tweet id: %s" % tweet.id)

	return render_template('tweets.html', data=data, pt=pt)	
예제 #3
0
def collection(name):

	# deal with message
	msg = utils.msgHandle(session)

	data = {}

	c = models.Collection.objects.get(name=name)
	data['collection'] = c

	# stringify search terms
	data['collection']['search_terms'] = ", ".join(data['collection']['search_terms'])

	# get job cron
	try:
		job = mycron.find_comment('twitore_{name}'.format(name=name)).next()
		logging.debug(job.command)
	except:
		logging.warning('could not find cron')

	return render_template('collection.html', data=data, msg=msg)	
예제 #4
0
def create_collection():

	data = {}

	if request.method =="GET":
		return render_template("create_collection.html")

	if request.method == "POST":
		logging.debug(request.form)

		# create collection
		collection = request.form['name']

		# check for collection name
		try:
			c = models.Collection.objects.get(name=collection)        
			return jsonify({'status':False})
		except DoesNotExist:
			logging.debug("collection does not exist, continuing")


		c = models.Collection()
		c.name = request.form['name']
		c.search_terms = [term.strip() for term in request.form['search_terms'].split(",")]
		c.minute_frequency = int(request.form['minute_frequency'])
		c.save()
		logging.debug("created collection %s" % c.id)

		# set cron job
		utils.setCron(mycron,c)

		# set msg in session		
		utils.setMsg(session,"collection created","msg_success")

		return redirect("/{prefix}/collections".format(prefix=localConfig.twitore_app_prefix))
예제 #5
0
def collections():

	# deal with message
	msg = utils.msgHandle(session)

	data = []
	collections = models.Collection.objects()

	'''
	This needs to be paginated
	'''
	for collection in collections:
		s = models.MongoTweet.objects().filter(twitore_collection=collection.name)
		data.append({
			"count":s.count(),
			"name":collection.name,
			'minute_frequency':collection.minute_frequency			
		})

	logging.debug(mycron.user)
	logging.debug(mycron.crons)

	return render_template('collections.html', data=data, msg=msg)
예제 #6
0
def update_collection(name):

	data = {}

	# assuming POST
	logging.debug(request.form)	

	# check for collection name
	try:
		c = models.Collection.objects.get(name=name)        
	except DoesNotExist:
		logging.debug("collection does not exist, continuing")
		return jsonify({'status':False})

	c.search_terms = [term.strip() for term in request.form['search_terms'].split(",")]
	c.minute_frequency = int(request.form['minute_frequency'])
	c.save()
	logging.debug("updated collection %s" % c.id)

	# update cron job
	job = mycron.find_comment('twitore_{name}'.format(name=name))
	job = list(job)
	logging.debug(job)

	# update cron
	'''
	Consider improving the logic here
	'''
	if len(job) > 0:
		if c.minute_frequency == 0:
			utils.deleteCron(mycron, job[0], c)
		elif c.minute_frequency >= 1 and c.minute_frequency <= 59:
			utils.updateCron(mycron,job[0],c)
		else:
			logging.warning('frequency must be between 1-59 minutes. aborting.')
			utils.setMsg(session,"collection could not be updated. frequency must be between 1-59 minutes. aborted.","msg_alert")
			return redirect("/{prefix}/collection/{name}".format(prefix=localConfig.twitore_app_prefix,name=name))
	
	# set cron if not there
	else:
		# set cron job
		utils.setCron(mycron,c)

	# set msg in session		
	utils.setMsg(session,"collection updated","msg_success")

	return redirect("/{prefix}/collection/{name}".format(prefix=localConfig.twitore_app_prefix,name=name))
예제 #7
0
	def burstStream(self):
		logging.debug("sending all local tweets as burst")
		for tweet in self.lt:
			tweet_process.delay(tweet)
예제 #8
0
	def captureStream(self):
		while True:
			logging.debug("sending stream at slight delay")
			for tweet in self.lt:
				tweet_process.delay(tweet)
				time.sleep(.25)
예제 #9
0
	def burstStream(self):
		logging.debug("sending all local tweets as burst")
		for tweet_json in self.lt:
			result = producer.send_messages("betweezered", tweet_json)
예제 #10
0
	def captureStream(self):
		while True:
			logging.debug("sending stream at slight delay")
			for tweet_json in self.lt:
				result = producer.send_messages("betweezered", tweet_json)
				time.sleep(.25)
예제 #11
0
	def consume(self):
		messages = self.consumer.fetch_messages()
		for message in messages:
			logging.debug("Kafka message received")
			result = self.processMessage(message)
예제 #12
0
def search(collection):

	'''
	Consider pushing to worker queue (celery, rq, etc.)
	'''

	logging.info("Performing search for %s" % collection)
	
	# retrieve collection mongo record
	try:
		c = models.Collection.objects.get(name=collection)
		logging.info("Retrieved %s %s" % (c.name,c.id))
	except DoesNotExist:
		logging.info("collection does not exist")
		return jsonify({"status":False})


	# run search (where collection name is used for )
	archive_dir = "/".join([localConfig.archive_directory,c.name])
	# search_terms = ",".join(c.search_terms)
	logging.debug("Passing %s %s" % (c.search_terms,archive_dir))
	archive_log = utils.search_and_archive(collection, c.search_terms, archive_dir)


	return jsonify({"archive_log":archive_log})























# # return json for tweet
# @app.route("/{prefix}/tweets/<limit>".format(prefix=localConfig.twitore_app_prefix), methods=['GET', 'POST'])
# def tweet(limit):

# 	renderdict = {}

# 	# return tweet json
# 	renderdict['tweets'] = models.MongoTweet.objects().limit(int(limit))
# 	renderdict['count'] = models.MongoTweet.objects.count()
# 	renderdict['limit'] = limit

# 	# add search terms
# 	renderdict['search_terms'] = localConfig.search_terms

# 	return render_template('tweets.htm',renderdict=renderdict)



# # cron
# @app.route("/{prefix}/jobs".format(prefix=localConfig.twitore_app_prefix), methods=['GET', 'POST'])
# def jobs():

# 	# get crontab for current user
# 	mycron = CronTab(user=True)

# 	# get all jobs
# 	jobs = mycron.crons
# 	localConfig.logging.debug(jobs)

# 	#render page
# 	return render_template('jobs.html',jobs=jobs)
예제 #13
0
 def burstStream(self):
     logging.debug("sending all local tweets as burst")
     for tweet in self.lt:
         tweet_process.delay(tweet)
예제 #14
0
 def captureStream(self):
     while True:
         logging.debug("sending stream at slight delay")
         for tweet in self.lt:
             tweet_process.delay(tweet)
             time.sleep(.25)
예제 #15
0
 def burstStream(self):
     logging.debug("sending all local tweets as burst")
     for tweet_json in self.lt:
         result = producer.send_messages("betweezered", tweet_json)
예제 #16
0
 def captureStream(self):
     while True:
         logging.debug("sending stream at slight delay")
         for tweet_json in self.lt:
             result = producer.send_messages("betweezered", tweet_json)
             time.sleep(.25)