def import_feed(request, pk):
feed = RSS_Feed.objects.get(pk=pk)
# add to queue
last_imported = datetime.datetime.now()
index_rss.apply_async( kwargs={ 'uri': feed.uri }, queue='tasks', priority=5 )
# save new timestamp
feed.last_imported = last_imported
feed.save()
return render(request, 'rss_manager/rss_feed_import.html', {'id': pk,})
def queue_index_rss(request):
uri = request.GET["uri"]
result = index_rss.apply_async(kwargs={'uri': uri},
queue='tasks',
priority=5)
return HttpResponse(json.dumps({'queue': result.id}),
content_type="application/json")
def queue_index_rss(request):
uri = request.GET["uri"]
from opensemanticetl.tasks import index_rss
result = index_rss.apply_async(kwargs={'uri': uri},
queue='open_semantic_etl_tasks',
priority=5)
return HttpResponse(json.dumps({'queue': result.id}),
content_type="application/json")
def import_feeds(request):
verbose = True
log = []
count = 0
count_queued = 0
for feed in RSS_Feed.objects.all():
count += 1
if verbose:
log.append( "Checking delta time of feed: {}".format(feed) )
add_to_queue = True
# If delta 0, no automatic import
if not feed.delta:
add_to_queue = False
# If delta time, do not import this feed within this time by setting add_to_queue to false
if feed.delta and feed.last_imported:
# when next import allowed (because time delta passed)?
next_import = feed.last_imported + timedelta(minutes=feed.delta)
# don't check time delta if last import in future (i.e. if system time was wrong)
if feed.last_imported < timezone.now():
# if time for next import not reached, do not index
if timezone.now() < next_import:
add_to_queue = False
if verbose:
log.append( "Last addition to queue: {}".format(feed.last_imported) )
log.append( "Next addition to queue: {}".format(next_import) )
if add_to_queue:
if verbose:
log.append( "Adding feed to queue: {}".format(feed) )
# add to queue
last_imported = datetime.datetime.now()
from opensemanticetl.tasks import index_rss
index_rss.apply_async( kwargs={ 'uri': feed.uri }, queue='open_semantic_etl_tasks', priority=5 )
# save new timestamp
feed.last_imported = last_imported
feed.save()
count_queued += 1
#
# stats / log
#
response = "Feeds to queue: {} of {}".format(count_queued, count)
if len(log) > 0:
response += "\n\n" + "\n".join(log)
#
# return response
#
status = HttpResponse(response)
status["Content-Type"] = "text/plain"
return status