def submit_edit(request, access_code):
user = User.objects.get(access_code=access_code)
url = get_url(request.GET.get('url'))
original_nl = get_nl(request.GET.get('original_nl'))
original_command = get_command(request.GET.get('original_command'))
nl = get_nl(request.GET.get('nl'))
command = get_command(request.GET.get('command'))
Annotation.objects.filter(url=url, nl=original_nl, cmd=original_command).delete()
annotation = Annotation.objects.create(url=url, nl=nl, cmd=command, annotator=user)
resp = json_response({'nl': annotation.nl.str, 'command': annotation.cmd.str},
status='EDIT_SAVED')
return resp
def example_requests_with_translations(request):
example_requests_with_translations = []
example_request_list = [
'remove all pdfs in my current directory',
'delete all *.txt files in "myDir/"',
'list files in "myDir/" that have been modified within 24 hours',
'find all files named "test*.cpp" and move them to "project/code/"',
'find all files larger than a gigabyte in the current folder',
'find all png files larger than 50M that were last modified more than 30 days ago'
]
for request_str in example_request_list:
nl = get_nl(request_str)
if Translation.objects.filter(nl__str=request_str).exists():
translations = Translation.objects.filter(nl__str=request_str)
max_score = translations.aggregate(Max('score'))['score__max']
for top_translation in Translation.objects.filter(
nl__str=request_str, score=max_score):
break
else:
# Compute the translations on the fly
top_translation = None
if not WEBSITE_DEVELOP:
# call learning model and store the translations
batch_outputs, output_logits = translate_fun(request_str)
max_score = -np.inf
if batch_outputs:
top_k_predictions = batch_outputs[0]
top_k_scores = output_logits[0]
for i in range(len(top_k_predictions)):
pred_tree, pred_cmd = top_k_predictions[i]
score = top_k_scores[i]
if score > max_score:
max_score = score
cmd = get_command(pred_cmd)
top_translation = Translation.objects.create(
nl=nl, pred_cmd=cmd, score=score)
if top_translation:
example_requests_with_translations.append({
'nl':
nl.str,
'top_translation':
top_translation.pred_cmd.str,
'tags': [
tag.str for tag in
top_translation.pred_cmd.tags.all().order_by('frequency')
]
})
else:
example_requests_with_translations.append({
'nl': nl.str,
'top_translation': 'No translation available.',
'tags': []
})
return json_response({
'example_requests_with_translations':
example_requests_with_translations
})
def mark_i_dont_know(request, access_code):
user = User.objects.get(access_code=access_code)
url = get_url(request.GET.get('url'))
cmd = get_command(request.GET.get('command'))
url.commands.remove(cmd);
Annotation.objects.create(
url=url, nl=get_nl('I DON\'T KNOW'), cmd=cmd, annotator=user)
return json_response(status='MARK_I_DONT_KNOW_SUCCESS')
def mark_wrong(request, access_code):
user = User.objects.get(access_code=access_code)
url = get_url(request.GET.get('url'))
cmd = get_command(request.GET.get('command'))
url.commands.remove(cmd);
Annotation.objects.create(
url=url, nl=get_nl('ERROR'), cmd=cmd, annotator=user)
return json_response(status='MARK_WRONG_SUCCESS')
def delete_annotation(request, access_code):
url = get_url(request.GET.get('url'))
nl = get_nl(request.GET.get('nl'))
command = get_command(request.GET.get('command'))
for annotation in Annotation.objects.filter(url=url, nl=nl, cmd=command):
for tag in command.tags.all():
tag.annotations.filter(id=annotation.id)
tag.save()
annotation.delete()
return json_response(status='DELETION_SUCCESS')
def submit_annotation(request, access_code):
user = User.objects.get(access_code=access_code)
url = get_url(request.GET.get('url'))
nl = get_nl(request.GET.get('nl'))
tag = get_tag(request.GET.get('utility'))
command = get_command(request.GET.get('command'))
url.tags.add(tag)
annotation = Annotation.objects.create(
url=url, nl=nl, cmd=command, annotator=user)
tag.annotations.add(annotation)
if not AnnotationProgress.objects.filter(annotator=user, url=url, tag=tag):
AnnotationProgress.objects.create(
annotator=user, url=url, tag=tag, status='in-progress')
resp = json_response({'nl': annotation.nl.str, 'command': annotation.cmd.str},
status='ANNOTATION_SAVED')
return resp
def load_commands_in_url(stackoverflow_dump_path):
url_prefix = 'https://stackoverflow.com/questions/'
with sqlite3.connect(stackoverflow_dump_path,
detect_types=sqlite3.PARSE_DECLTYPES) as db:
for url in URL.objects.all():
# url = URL.objects.get(str='https://stackoverflow.com/questions/127669')
url.commands.clear()
print(url.str)
for answer_body, in db.cursor().execute("""
SELECT answers.Body FROM answers
WHERE answers.ParentId = ?""", (url.str[len(url_prefix):],)):
url.html_content = answer_body
for code_block in extract_code(url.html_content):
for cmd in extract_oneliners_from_code(code_block):
ast = data_tools.bash_parser(cmd)
if ast:
command = get_command(cmd)
print('extracted: {}'.format(cmd))
url.commands.add(command)
url.save()
def translate(request, ip_address):
template = loader.get_template('translator/translate.html')
if request.method == 'POST':
request_str = request.POST.get('request_str')
else:
request_str = request.GET.get('request_str')
if not request_str or not request_str.strip():
return redirect('/')
while request_str.endswith('/'):
request_str = request_str[:-1]
# check if the natural language request is in the database
nl = get_nl(request_str)
trans_list = []
annotated_trans_list = []
if CACHE_TRANSLATIONS and \
Translation.objects.filter(nl=nl).exists():
# model translations exist
cached_trans = Translation.objects.filter(nl=nl).order_by('score')
count = 0
for trans in cached_trans:
pred_tree = data_tools.bash_parser(trans.pred_cmd.str)
if pred_tree is not None:
trans_list.append(trans)
annotated_trans_list.append(tokens2html(pred_tree))
count += 1
if count >= NUM_TRANSLATIONS:
break
# check if the user is in the database
try:
user = User.objects.get(ip_address=ip_address)
except ObjectDoesNotExist:
if ip_address == '123.456.789.012':
organization = ''
city = '--'
region = '--'
country = '--'
else:
r = requests.get('http://ipinfo.io/{}/json'.format(ip_address))
organization = '' if r.json()['org'] is None else r.json()['org']
city = '--' if r.json()['city'] is None else r.json()['city']
region = '--' if r.json()['region'] is None else r.json()['region']
country = '--' if r.json()['country'] is None else r.json(
)['country']
user = User.objects.create(ip_address=ip_address,
organization=organization,
city=city,
region=region,
country=country)
# save the natural language request issued by this IP Address
nl_request = NLRequest.objects.create(nl=nl, user=user)
if not trans_list:
if not WEBSITE_DEVELOP:
# call learning model and store the translations
batch_outputs, output_logits = translate_fun(request_str)
if batch_outputs:
top_k_predictions = batch_outputs[0]
top_k_scores = output_logits[0]
for i in range(len(top_k_predictions)):
pred_tree, pred_cmd = top_k_predictions[i]
score = top_k_scores[i]
cmd = get_command(pred_cmd)
trans_set = Translation.objects.filter(nl=nl, pred_cmd=cmd)
if not trans_set.exists():
trans = Translation.objects.create(nl=nl,
pred_cmd=cmd,
score=score)
else:
for trans in trans_set:
break
trans.score = score
trans.save()
trans_list.append(trans)
start_time = time.time()
annotated_trans_list.append(tokens2html(pred_tree))
print(time.time() - start_time)
start_time = time.time()
translation_list = []
for trans, annotated_cmd in zip(trans_list, annotated_trans_list):
upvoted, downvoted, starred = "", "", ""
if Vote.objects.filter(translation=trans,
ip_address=ip_address).exists():
v = Vote.objects.get(translation=trans, ip_address=ip_address)
upvoted = 1 if v.upvoted else ""
downvoted = 1 if v.downvoted else ""
starred = 1 if v.starred else ""
translation_list.append(
(trans, upvoted, downvoted, starred,
trans.pred_cmd.str.replace('\\', '\\\\'), annotated_cmd))
# sort translation_list based on voting results
translation_list.sort(key=lambda x: x[0].num_votes + x[0].score,
reverse=True)
context = {'nl_request': nl_request, 'trans_list': translation_list}
return HttpResponse(template.render(context, request))
