def score_functions(request):
#TODO: make it part of index/package configuration
account = request.user.get_profile().account
if request.method == 'GET':
index_code = request.GET['index_code']
index = Index.objects.get(code=index_code)
functions = get_functions(index)
form = ScoreFunctionForm()
context = {
'form': form,
'account': request.user.get_profile().account,
'navigation_pos': 'dashboard',
'functions': functions,
'index_code': index_code,
'functions_available': len(functions) < functions_number,
}
return render('score_functions.html', request, context_dict=context)
else:
form = ScoreFunctionForm(data=request.POST)
if form.is_valid():
index_code = request.POST['index_code']
index = Index.objects.get(code=index_code)
name = form.cleaned_data['name']
definition = form.cleaned_data['definition']
client = ApiClient(account.get_private_apiurl()).get_index(
index.name)
try:
client.add_function(int(name), definition)
except InvalidDefinition, e:
index = Index.objects.get(code=index_code)
functions = get_functions(index)
form = ScoreFunctionForm(initial={
'name': name,
'definition': definition
})
messages.error(request, 'Problem processing your formula: %s',
str(e))
context = {
'form': form,
'account': request.user.get_profile().account,
'navigation_pos': 'dashboard',
'functions': functions,
'index_code': index_code,
'functions_available': len(functions) < functions_number,
}
return render('score_functions.html',
request,
context_dict=context)
def delete_index(request):
if request.method == 'GET':
return HttpResponseRedirect(reverse('dashboard'))
else:
index = Index.objects.get(id=request.POST['index_id'])
index_client = ApiClient(index.account.get_private_apiurl()).get_index(index.name)
index_client.delete_index()
return HttpResponseRedirect(reverse('dashboard'))
def remove_function(request):
account = request.user.get_profile().account
if request.method == 'GET':
index_code = request.GET['index_code']
index = Index.objects.get(code=index_code)
function_name = request.GET['function_name']
client = ApiClient(account.get_private_apiurl()).get_index(index.name)
client.delete_function(function_name)
return HttpResponseRedirect(reverse('score_functions') + '?index_code=' + index_code)
def remove_function(request):
account = request.user.get_profile().account
if request.method == 'GET':
index_code = request.GET['index_code']
index = Index.objects.get(code=index_code)
function_name = request.GET['function_name']
client = ApiClient(account.get_private_apiurl()).get_index(index.name)
client.delete_function(function_name)
return HttpResponseRedirect(
reverse('score_functions') + '?index_code=' + index_code)
def delete_index(request):
if request.method == 'GET':
return HttpResponseRedirect(reverse('dashboard'))
else:
index = Index.objects.get(id=request.POST['index_id'])
index_client = ApiClient(index.account.get_private_apiurl()).get_index(
index.name)
index_client.delete_index()
return HttpResponseRedirect(reverse('dashboard'))
def _populate_batch(self, index, dataset, line_from, lines):
logger.info('Adding batch from line ' + str(line_from))
client = ApiClient(index.account.get_private_apiurl())
index = client.get_index(index.name)
try:
dataset_file = open(dataset_files_path + dataset.filename, 'r')
for i in range(line_from):
dataset_file.readline()
except Exception, e:
logger.error('Failed processing dataset file: %s', e)
return False, 0
def _populate_batch(self, index, dataset, line_from, lines):
logger.info('Adding batch from line ' + str(line_from))
client = ApiClient(index.account.get_private_apiurl())
index = client.get_index(index.name)
try:
dataset_file = open(dataset_files_path + dataset.filename, 'r')
for i in range(line_from):
dataset_file.readline()
except Exception, e:
logger.error('Failed processing dataset file: %s', e)
return False, 0
def manage_index(request, index_code=None):
account = request.user.get_profile().account
index = Index.objects.get(code=index_code)
if index:
if index.account == account:
if request.method == 'GET':
index = Index.objects.get(code=index_code)
largest_func = max(
[int(f.name) + 1
for f in index.scorefunctions.all()] + [5])
functions = get_functions(index, upto=largest_func)
context = {
'account': request.user.get_profile().account,
'navigation_pos': 'dashboard',
'functions': functions,
'index': index,
'index_code': index_code,
'largest_func': largest_func
}
if 'query' in request.GET:
maxim = int(request.GET.get('max', '25'))
index_client = ApiClient(
account.get_private_apiurl()).get_index(index.name)
context['results'] = index_client.search(
request.GET['query'], length=max)
context['query'] = request.GET['query']
context['more'] = maxim + 25
return render('manage_index.html',
request,
context_dict=context)
else:
if 'definition' in request.POST:
name = request.POST['name']
definition = request.POST['definition']
client = ApiClient(account.get_private_apiurl()).get_index(
index.name)
try:
if definition:
client.add_function(int(name), definition)
else:
client.delete_function(int(name))
except InvalidDefinition, e:
return HttpResponse('Invalid function', status=400)
return JsonResponse({'largest': 5})
elif 'public_api' in request.POST:
index.public_api = request.POST['public_api'] == 'true'
index.save()
return JsonResponse({'public_api': index.public_api})
def score_functions(request):
#TODO: make it part of index/package configuration
account = request.user.get_profile().account
if request.method == 'GET':
index_code = request.GET['index_code']
index = Index.objects.get(code=index_code)
functions = get_functions(index)
form = ScoreFunctionForm()
context = {
'form': form,
'account': request.user.get_profile().account,
'navigation_pos': 'dashboard',
'functions': functions,
'index_code': index_code,
'functions_available': len(functions) < functions_number,
}
return render('score_functions.html', request, context_dict=context)
else:
form = ScoreFunctionForm(data=request.POST)
if form.is_valid():
index_code = request.POST['index_code']
index = Index.objects.get(code=index_code)
name = form.cleaned_data['name']
definition = form.cleaned_data['definition']
client = ApiClient(account.get_private_apiurl()).get_index(index.name)
try:
client.add_function(int(name), definition)
except InvalidDefinition, e:
index = Index.objects.get(code=index_code)
functions = get_functions(index)
form = ScoreFunctionForm(initial={'name': name, 'definition': definition})
messages.error(request, 'Problem processing your formula: %s', str(e))
context = {
'form': form,
'account': request.user.get_profile().account,
'navigation_pos': 'dashboard',
'functions': functions,
'index_code': index_code,
'functions_available': len(functions) < functions_number,
}
return render('score_functions.html', request, context_dict=context)
def create_index(request):
account = request.user.get_profile().account
if request.method == 'GET':
index_qty = len(account.indexes.all())
default_name = '' #'Index_' + str(index_qty + 1)
form = IndexForm(initial={'name': default_name})
context = {
'form': form,
'account': request.user.get_profile().account,
'navigation_pos': 'dashboard',
}
return render('new-index.html', request, context_dict=context)
else:
form = IndexForm(data=request.POST)
if form.is_valid():
try:
client = ApiClient(account.get_private_apiurl())
client.create_index(form.cleaned_data['name'])
messages.success(request, 'New index created successfully.')
except IndexAlreadyExists:
context = {
'form': form,
'account': request.user.get_profile().account,
'navigation_pos': 'dashboard',
}
messages.error(request,
'You already have an Index with that name.')
return render('new-index.html', request, context_dict=context)
except TooManyIndexes:
context = {
'form': form,
'account': request.user.get_profile().account,
'navigation_pos': 'dashboard',
}
messages.error(
request,
'You already have the maximum number of indexes allowed for your account. If you need more, please contact support.'
)
return render('new-index.html', request, context_dict=context)
except Exception, e:
print e
messages.error(
request,
'Unexpected error creating the index. Try again in a few minutes'
)
return HttpResponseRedirect(reverse('dashboard'))
else:
def create_index(request):
account = request.user.get_profile().account
if request.method == 'GET':
index_qty = len(account.indexes.all())
default_name = '' #'Index_' + str(index_qty + 1)
form = IndexForm(initial={'name': default_name})
context = {
'form': form,
'account': request.user.get_profile().account,
'navigation_pos': 'dashboard',
}
return render('new-index.html', request, context_dict=context)
else:
form = IndexForm(data=request.POST)
if form.is_valid():
try:
client = ApiClient(account.get_private_apiurl())
client.create_index(form.cleaned_data['name'])
messages.success(request, 'New index created successfully.')
except IndexAlreadyExists:
context = {
'form': form,
'account': request.user.get_profile().account,
'navigation_pos': 'dashboard',
}
messages.error(request, 'You already have an Index with that name.')
return render('new-index.html', request, context_dict=context)
except TooManyIndexes:
context = {
'form': form,
'account': request.user.get_profile().account,
'navigation_pos': 'dashboard',
}
messages.error(request, 'You already have the maximum number of indexes allowed for your account. If you need more, please contact support.')
return render('new-index.html', request, context_dict=context)
except Exception, e:
print e
messages.error(request, 'Unexpected error creating the index. Try again in a few minutes')
return HttpResponseRedirect(reverse('dashboard'))
else:
def monitor(self, population):
client = ApiClient(population.index.account.get_private_apiurl())
index = client.get_index(population.index.name)
if index.has_started():
if population.status == IndexPopulation.Statuses.created:
logger.info('Populating index ' + population.index.code + ' with dataset "' + population.dataset.name + '"')
population.status = IndexPopulation.Statuses.populating
population.populated_size = 0
population.save()
eof, documents_added = self._populate_batch(population.index, population.dataset, population.populated_size, batch_size)
population.populated_size = population.populated_size + documents_added
if eof:
population.status = IndexPopulation.Statuses.finished
population.save()
return True
def monitor(self, population):
client = ApiClient(population.index.account.get_private_apiurl())
index = client.get_index(population.index.name)
if index.has_started():
if population.status == IndexPopulation.Statuses.created:
logger.info('Populating index ' + population.index.code +
' with dataset "' + population.dataset.name + '"')
population.status = IndexPopulation.Statuses.populating
population.populated_size = 0
population.save()
eof, documents_added = self._populate_batch(
population.index, population.dataset,
population.populated_size, batch_size)
population.populated_size = population.populated_size + documents_added
if eof:
population.status = IndexPopulation.Statuses.finished
population.save()
return True
def manage_index(request, index_code=None):
account = request.user.get_profile().account
index = Index.objects.get(code=index_code)
if index:
if index.account == account:
if request.method == 'GET':
index = Index.objects.get(code=index_code)
largest_func = max([int(f.name) + 1 for f in index.scorefunctions.all()] + [5])
functions = get_functions(index, upto=largest_func)
context = {
'account': request.user.get_profile().account,
'navigation_pos': 'dashboard',
'functions': functions,
'index': index,
'index_code': index_code,
'largest_func': largest_func
}
if 'query' in request.GET:
maxim = int(request.GET.get('max', '25'))
index_client = ApiClient(account.get_private_apiurl()).get_index(index.name)
context['results'] = index_client.search(request.GET['query'], length=max)
context['query'] = request.GET['query']
context['more'] = maxim + 25
return render('manage_index.html', request, context_dict=context)
else:
if 'definition' in request.POST:
name = request.POST['name']
definition = request.POST['definition']
client = ApiClient(account.get_private_apiurl()).get_index(index.name)
try:
if definition:
client.add_function(int(name), definition)
else:
client.delete_function(int(name))
except InvalidDefinition, e:
return HttpResponse('Invalid function', status=400)
return JsonResponse({'largest': 5})
elif 'public_api' in request.POST:
index.public_api = request.POST['public_api'] == 'true'
index.save()
return JsonResponse({'public_api': index.public_api})
def drop_index(self, index):
client = ApiClient(self.get_private_apiurl())
client.delete_index(index.name)
def drop_index(self, index):
client = ApiClient(self.get_private_apiurl())
client.delete_index(index.name)
from lib.indextank.client import ApiClient
import sys
import time
if len(sys.argv) != 2:
print 'Usage: testapi.py <API_URL>'
api = ApiClient(sys.argv[1])
index = api.get_index('testapi.py')
if index.exists():
print 'deleting previous index'
index.delete_index()
print 'creating index'
index.create_index()
print 'waiting to start...'
while not index.has_started():
time.sleep(1)
print 'adding docs'
index.add_document(1, {'text': 'a b c1'}, variables={0: 1, 1: 2, 2: 3})
index.add_document(2, {'text': 'a b c2'}, variables={0: 2, 1: 2, 2: 2})
index.add_document(3, {'text': 'a b c3'}, variables={0: 3, 1: 2, 2: 1})
print 'adding functions'
index.add_function(1, 'd[0]')
index.add_function(2, 'd[2]')
print 'checking functions'
from lib.indextank.client import ApiClient
import sys
import time
if len(sys.argv) != 2:
print "Usage: testapi.py <API_URL>"
api = ApiClient(sys.argv[1])
index = api.get_index("testapi.py")
if index.exists():
print "deleting previous index"
index.delete_index()
print "creating index"
index.create_index()
print "waiting to start..."
while not index.has_started():
time.sleep(1)
print "adding docs"
index.add_document(1, {"text": "a b c1"}, variables={0: 1, 1: 2, 2: 3})
index.add_document(2, {"text": "a b c2"}, variables={0: 2, 1: 2, 2: 2})
index.add_document(3, {"text": "a b c3"}, variables={0: 3, 1: 2, 2: 1})
print "adding functions"
index.add_function(1, "d[0]")
index.add_function(2, "d[2]")
print "checking functions"
