def select(ast, utility_set):
for ut in data_tools.get_utilities(ast):
if not ut in utility_set:
print('Utility currently not handled: {} - {}'.format(
ut, data_tools.ast2command(ast, loose_constraints=True)))
return False
return True
def compute_top_utilities(path, k):
print('computing top most frequent utilities...')
utilities = collections.defaultdict(int)
with open(path) as f:
while (True):
command = f.readline().strip()
if not command:
break
ast = data_tools.bash_parser(command, verbose=False)
for u in data_tools.get_utilities(ast):
utilities[u] += 1
top_utilities = []
freq_threshold = -1
for u, freq in sorted(utilities.items(), key=lambda x: x[1], reverse=True):
if freq_threshold > 0 and freq < freq_threshold:
break
# Xingyu Wei: To udpate for word assignment and control flow,
# we remove the Grey List
if u in bash.BLACK_LIST:
continue
top_utilities.append(u)
print('{}: {} ({})'.format(len(top_utilities), u, freq))
if len(top_utilities) == k:
freq_threshold = freq
top_utilities = set(top_utilities)
return top_utilities
def run():
sqlite_filename = sys.argv[1]
url_prefix = 'https://stackoverflow.com/questions/'
urls = {}
commands = {}
with sqlite3.connect(sqlite_filename, detect_types=sqlite3.PARSE_DECLTYPES) as db:
count = 0
for post_id, answer_body in db.cursor().execute("""
SELECT questions.Id, answers.Body FROM questions, answers
WHERE questions.Id = answers.ParentId
ORDER BY questions.Score DESC"""):
print(post_id)
for code_block in extract_code(answer_body):
for cmd in extract_oneliner_from_code(code_block):
print('command string: {}'.format(cmd))
ast = data_tools.bash_parser(cmd)
if not ast:
continue
utilities = data_tools.get_utilities(ast)
for utility in utilities:
if utility in bash.top_100_utilities:
print('extracted: {}, {}'.format(utility, cmd))
temp = data_tools.ast2template(ast, loose_constraints=True)
if not utility in commands:
commands[utility] = {}
commands[utility][temp] = cmd
urls[utility] = {'{}{}'.format(url_prefix, post_id)}
else:
if len(commands[utility]) >= NUM_COMMAND_THRESHOLD:
continue
if not temp in commands[utility]:
commands[utility][temp] = cmd
urls[utility].add('{}{}'.format(url_prefix, post_id))
count += 1
if count % 1000 == 0:
completed = False
for utility in bash.top_100_utilities:
if not utility in commands or len(commands[utility]) < NUM_COMMAND_THRESHOLD:
completed = False
else:
print('{} collection done.'.format(utility))
if completed:
break
with open('stackoverflow.urls', 'wb') as o_f:
pickle.dump(urls, o_f)
with open('stackoverflow.commands', 'wb') as o_f:
pickle.dump(commands, o_f)
for utility in commands:
print('{} ({})'.format(utility, len(commands[utility])))
for cmd in commands[utility]:
print(cmd)
def get_u_hist_from_file(input_file):
u_hist = collections.defaultdict(int)
with open(input_file) as f:
for cmd in f:
ast = data_tools.bash_parser(cmd, verbose=False)
for u in data_tools.get_utilities(ast):
if u in bash.BLACK_LIST or u in bash.GREY_LIST:
continue
u_hist[u] += 1
return u_hist
def populate_command_tags():
for cmd in Command.objects.all():
if len(cmd.str) > 600:
cmd.delete()
else:
cmd.tags.clear()
print(cmd.str)
ast = data_tools.bash_parser(cmd.str)
for utility in data_tools.get_utilities(ast):
print(utility)
cmd.tags.add(get_tag(utility))
cmd.save()
def get_command(command_str):
command_str = command_str.strip()
if Command.objects.filter(str=command_str).exists():
cmd = Command.objects.get(str=command_str)
else:
cmd = Command.objects.create(str=command_str)
ast = data_tools.bash_parser(command_str)
for utility in data_tools.get_utilities(ast):
cmd.tags.add(get_tag(utility))
template = data_tools.ast2template(ast, loose_constraints=True)
cmd.template = template
cmd.save()
return cmd
def compute_flag_stats():
input_file = sys.argv[1]
train_file = sys.argv[2]
u_hist = collections.defaultdict(int)
with open(input_file) as f:
for cmd in f:
ast = data_tools.bash_parser(cmd, verbose=False)
for u in data_tools.get_utilities(ast):
if u in bash.BLACK_LIST or u in bash.GREY_LIST:
continue
u_hist[u] += 1
sorted_u_by_freq = sorted(u_hist.items(), key=lambda x: x[1], reverse=True)
most_frequent_10 = [u for u, _ in sorted_u_by_freq[:10]]
least_frequent_10 = [u for u, _ in sorted_u_by_freq[-10:]]
most_frequent_10_flags = collections.defaultdict(set)
least_frequent_10_flags = collections.defaultdict(set)
with open(train_file) as f:
for cmd in f:
tokens = data_tools.bash_tokenizer(cmd,
loose_constraints=True,
with_flag_head=True)
for token in tokens:
if '@@' in token:
u, f = token.split('@@')
if u in most_frequent_10:
most_frequent_10_flags[u].add(f)
if u in least_frequent_10:
least_frequent_10_flags[u].add(f)
for u in most_frequent_10:
if u in most_frequent_10_flags:
print(u, data_tools.get_utility_statistics(u),
len(most_frequent_10_flags[u]))
else:
print(u, data_tools.get_utility_statistics(u), 0)
print()
for u in least_frequent_10:
if u in least_frequent_10_flags:
print(u, data_tools.get_utility_statistics(u),
len(least_frequent_10_flags[u]))
else:
print(u, data_tools.get_utility_statistics(u), 0)
def u_hist_to_radar_chart():
input_file = sys.argv[1]
u_hist = collections.defaultdict(int)
with open(input_file) as f:
for cmd in f:
ast = data_tools.bash_parser(cmd, verbose=False)
for u in data_tools.get_utilities(ast):
if u in bash.BLACK_LIST or u in bash.GREY_LIST:
continue
u_hist[u] += 1
selected_utilities = []
for i, (u, freq) in enumerate(
sorted(u_hist.items(), key=lambda x: x[1], reverse=True)):
if i >= 50:
print('{{axis:"{}",value:{:.2f}}},'.format(u, freq))
selected_utilities.append(u)
print()
def print_error_analysis_csv(grouped_dataset,
prediction_list,
FLAGS,
cached_evaluation_results=None,
group_by_utility=False,
error_predictions_only=True):
"""
Convert dev/test set examples to csv format so as to make it easier for
human annotators to enter their judgements.
:param grouped_dataset: dev/test set grouped by natural language.
:param prediction_list: model predictions.
:param FLAGS: experiment hyperparameters.
:param cached_evaluation_results: cached evaluation results from previous
rounds.
:param group_by_utility: if set, group the error examples by the utilities
used in the ground truth.
"""
def mark_example(error_list, example, gt_utility=None):
if gt_utility:
error_list[gt_utility].append(example)
else:
error_list.append(example)
eval_bash = FLAGS.dataset.startswith("bash")
cmd_parser = data_tools.bash_parser if eval_bash \
else data_tools.paren_parser
if group_by_utility:
utility_index = {}
for line in bash.utility_stats.split('\n'):
ind, utility, _, _ = line.split(',')
utility_index[utility] = ind
grammar_errors = collections.defaultdict(list) if group_by_utility else []
argument_errors = collections.defaultdict(list) if group_by_utility else []
example_id = 0
for nl_temp, data_group in grouped_dataset:
sc_txt = data_group[0].sc_txt.strip()
sc_temp = get_example_nl_key(sc_txt)
tg_strs = [dp.tg_txt for dp in data_group]
gt_trees = [cmd_parser(cm_str) for cm_str in tg_strs]
if group_by_utility:
gt_utilities = functools.reduce(
lambda x, y: x | y,
[data_tools.get_utilities(gt) for gt in gt_trees])
gt_utility = sorted(list(gt_utilities),
key=lambda x: int(utility_index[x]))[-1]
else:
gt_utility = None
predictions = prediction_list[example_id]
example_id += 1
example = []
grammar_error, argument_error = False, False
for i in xrange(min(3, len(predictions))):
if i == 0:
output_str = '{},"{}",'.format(example_id,
sc_txt.replace('"', '""'))
else:
output_str = ',,'
pred_cmd = predictions[i]
tree = cmd_parser(pred_cmd)
# evaluation ignoring flag orders
temp_match = tree_dist.one_match(gt_trees,
tree,
ignore_arg_value=True)
str_match = tree_dist.one_match(gt_trees,
tree,
ignore_arg_value=False)
if i < len(tg_strs):
output_str += '"{}",'.format(tg_strs[i].strip().replace(
'"', '""'))
else:
output_str += ','
output_str += '"{}",'.format(pred_cmd.replace('"', '""'))
if not str_match:
if temp_match:
if i == 0:
argument_error = True
grammar_error = True
else:
if i == 0:
grammar_error = True
example_sig = '{}<NL_PREDICTION>{}'.format(sc_temp, pred_cmd)
if cached_evaluation_results and \
example_sig in cached_evaluation_results:
output_str += cached_evaluation_results[example_sig]
else:
if str_match:
output_str += 'y,y'
elif temp_match:
output_str += 'y,'
example.append(output_str)
if error_predictions_only:
if grammar_error:
mark_example(grammar_errors, example, gt_utility)
elif argument_error:
mark_example(argument_errors, example, gt_utility)
else:
mark_example(grammar_errors, example, gt_utility)
return grammar_errors, argument_errors