def logic_type(cell, max_api_seq_len, min_api_seq_len, key): if is_boilerplate(cell): # boilerplate may not tokenize so leave it return 'boilerplate' try: toks, types = tokenize_and_templatize(cell[key]) except: return 'untokenizeable' api_seq = gen_api_seq(toks, types) if len(api_seq) > max_api_seq_len: return f'api seq longer than {max_api_seq_len}' if len(api_seq) < min_api_seq_len: return f'api seq shorter than {min_api_seq_len}' if 'class' in toks: return 'class' elif toks.count('def') > 1: return 'more than 1 function' elif toks.count('def') == 1: return '1 function' else: return 'pure logic'
def convert2to3(cell): original_code = cell['source'] try: x = ast.parse(original_code) y = tokenize_and_templatize(original_code) return cell except: pass # fixers = ['lib2to3.fixes.fix_print'] # refactor = RefactoringTool(fixer_names=fixers) # tree = refactor.refactor_string(original_code, 'temp') try: # fixers = get_fixers_from_package('lib2to3.fixes') # the only python2 issue is print statements so we only import this # fixer since it makes RefactorTool way faster fixers = ['lib2to3.fixes.fix_print'] refactor = RefactoringTool(fixer_names=fixers) tree = refactor.refactor_string(original_code, 'temp') converted_code = str(tree) print(converted_code) x = ast.parse(converted_code) # y = tokenize_and_templatize(converted_code) # print('===============\n', original_code, '\n', converted_code) cell['source'] = converted_code except Exception as e: print(e) # print('==========\n', print(original_code)) pass return cell
def get_imports(nb_cells, cell_index): # get all imports including from current cell imports = [] for c in nb_cells[:cell_index + 1]: if is_code(c): for line in c['source'].splitlines(): if (line.strip().startswith('import ') # some edge cases have from but no import token or (line.strip().startswith('from ') and 'import' in line)): # if 'import' not in line: # print(line) # raise ValueError(line) # we pick after import to save on total num tokens imports.append(line[line.index('import'):]) imports_string = '\n'.join(imports) import_toks = [] try: toks, types = tokenize_and_templatize(imports_string) # print(toks, types) for i, (tok, type) in enumerate(zip(toks, types)): if type in ['NAME', 'import', 'as', ',']: import_toks.append(tok) except: # the tokenization failed because of a multiline import using parenthesis import_toks = re.split('(\W+)', imports_string) # toks look like this: ['import', ' ', 'pyspark', '\n',] so we want to exclude # newlines and spaces import_toks = [t for t in import_toks if t.strip()] return import_toks
def does_parse(original_code, url=None): assert isinstance(original_code, str) try: x = ast.parse(original_code) y = tokenize_and_templatize(original_code) return True except: return False
def get_solution_new(group): '''We deduplicate solutions and pick most common solution.''' cells = [row.target_cell for _, row in group.iterrows()] solution2count = Counter() for c in cells: solution2count[c['code']] += 1 submission2count = Counter() for c in cells: submission2count[c['metadata']['boilerplate']] += 1 most_comm_sol = solution2count.most_common(1)[0][0] most_comm_sub = submission2count.most_common(1)[0][0] if (solution2count.most_common(1)[0][1] > 1 and submission2count.most_common(1)[0][1] > 1): top_solution_cells = [c for c in cells if c['code'] == most_comm_sol] # verify from different repositories repos = set([c['metadata']['repo'] for c in top_solution_cells]) if len(repos) < 2: return 'only from 1 repo' # we arbitrarily pick the first one target_cell = top_solution_cells[0] if not target_cell['code_tokens']: return 'empty target cell tokens' # Now we want to pick code blocks with an empty boilerplate. target_cell['boilerplate_code'] = most_comm_sub if most_comm_sub.strip(): try: toks, types = tokenize_and_templatize(most_comm_sub) except: return 'boiler tokenization failed' replaced = replace_newlines_indents(toks, types, enable_assert=False, comments=True) replaced = [t for t in replaced if t not in ['NEWLINE', 'INDENT']] if len(replaced) > 10: # i like 10 as a threshold since some spurious submissions are actually answers # and hence would be a spurious boilerplate. the main motivation is to allow comments # which would be removed by the replaced above. unfortunately empty function # boilerplates also removed return 'boilerplate longer than 10 toks' target_cell['boilerplate_code_tokens'] = [] return target_cell else: return 'only 1 solution'
def handle_solution(all_cells): ''' extract code within begin solution tag. requirement is that its non empty and multiple cells have the same solution.''' # filter cells with empty solution, these can become boilerplate!! implemented_solutions =[] for c in all_cells: if c['code_tokens']: if get_in_between_solution_comments(c['code_tokens']): implemented_solutions.append(c) del all_cells # dictionary of solution to cell indices sol2idx = defaultdict(list) for i, c in enumerate(implemented_solutions): sol2idx[c['code']].append(i) # we pick from most common for indices in sorted(list(sol2idx.values()), key=lambda x:len(x), reverse=True): if len(indices) < 2: return 'only 1 solution' target_cell = implemented_solutions[indices[0]] start, end = get_in_between_solution_comments(target_cell['code_tokens']) # We tokenize again to get the types toks, types = tokenize_and_templatize(target_cell['code']) target_cell['extracted_code_tokens'] = replace_newlines_indents(toks[start:end], types[start:end], enable_assert=False, strings=True,comments=True) target_cell['extracted_code_types'] = types[start:end] # Get the boilerplate tokens surrounding the solution. boiler_toks = toks[:start] + toks[end:] boiler_types = types[:start] + types[end:] target_cell['boilerplate_code_tokens'] = replace_newlines_indents(boiler_toks, boiler_types,enable_assert=False, strings=True,comments=True) target_cell['boilerplate_code_types'] = boiler_types return target_cell
def does_parse(original_code, url=None): assert isinstance(original_code, str) # lines with %matplotlib inline don't parse, we'll keep them for now # lines = [] # for line in original_code.splitlines(): # if not line.strip().startswith('%'): # lines.append(line) # code = '\n'.join(lines) try: x = ast.parse(original_code) y = tokenize_and_templatize(original_code) return True except: # todo deal with these ''' what gets removed: - slight indentation issue ''' # print('=========\n', original_code, url) return False
def get_all_code_process(code, allow_api_declarations=False): '''Lots of old code tokenization. We use "code_tokens_clean" ''' try: targ_tokens, types = tokenize_and_templatize(code) api_sequence = gen_api_seq(targ_tokens, types, allow_declarations=allow_api_declarations) comments, docstring = extract_comments(targ_tokens, types) # full code, with no extraneous newlines, and normalized strings. i plan to use this for # context and target code! clean_code = astor.to_source(ast.parse(code)) clean_code_toks, clean_code_types = tokenize_and_templatize(clean_code) clean_code_toks = replace_newlines_indents(clean_code_toks, clean_code_types, strings=True, enable_assert=False) # todo do a lstrip newlines on targ_tokens strip_import_or_func = take_single_function(code) strip_import_or_func_toks, strip_types = tokenize_and_templatize( strip_import_or_func) normalized = normalize_code(strip_import_or_func) normalized_tokens, norm_types = tokenize_and_templatize(normalized) except: targ_tokens = None clean_code_toks = None api_sequence = None normalized_tokens = None norm_types = None comments = None docstring = None strip_import_or_func_toks = None strip_types = None newline_tok = 'NEWLINE' if strip_import_or_func_toks: # here we do partial normalization. replace strings, but leave variable names. # this is particularly useful for context cells so we can copy initializations # from above code_tokens_with_vars_no_strings = replace_newlines_indents( strip_import_or_func_toks, strip_types, newline_tok, strings=True) code_tokens_with_vars_no_strings_trunc = truncate_code_to_lines( code_tokens_with_vars_no_strings, newline_tok, 10) else: code_tokens_with_vars_no_strings_trunc = None # we move this logic out of try since better to avoid try if possible if normalized_tokens: replaced = replace_newlines_indents(normalized_tokens, norm_types, newline_tok) normalized_trunc = truncate_code_to_lines(replaced, newline_tok, 10) if len(api_sequence) == 0: api_sequence = ['NO_API_SEQUENCE'] else: normalized_tokens = None normalized_trunc = None js = { 'code': code, 'code_tokens_normalized': normalized_tokens, 'code_tokens_normalized_trunc': normalized_trunc, 'code_tokens_partial_normalized_trunc': code_tokens_with_vars_no_strings_trunc, 'code_tokens': targ_tokens, 'code_tokens_clean': clean_code_toks, 'comments': comments, 'docstring': docstring, 'api_sequence': api_sequence, } return js
def dedup_boiler_extract(group, boiler_set): solutions = group[group.is_boilerplate==False] if not solutions.empty: # if group.is_boilerplate.any(): # return 'both' all_cells = [row.target_cell for _, row in solutions.iterrows()] # Instructor answers have a different extraction process. for c in all_cells: if c['is_instructor_answer']: return handle_solution(all_cells) # Get cells which passed test cases. passed_cells = [] for c in all_cells: if 'test_below_passed' in c['metadata'] and c['metadata']['test_below_passed']: # Prefer cells from output notebooks by adding to front. if 'from_output_nb' in c['metadata'] and c['metadata']['from_output_nb']: passed_cells.insert(0, c) else: passed_cells.append(c) # If no passed test cases omit examples. if not passed_cells: return 'no autograder tests or none passed' target_cell = passed_cells[0] # Retrieve boilerplate, either if its not in our dataset, we try to match # all the boilerplates or dataset contains with the cell. if group.is_boilerplate.any(): boiler = group[group.is_boilerplate].iloc[0].target_cell['code'] else: boiler = find_boilerplate_for_checksum(boiler_set, target_cell) if boiler: # Try to extract the student code from the solution, excluding boilerplate. extracted = diff_cell(target_cell['code'], boiler) if extracted: # boiler_js = get_all_code_process(boiler) # extract_js = get_all_code_process(extracted) target_cell['extracted_code'] = extracted target_cell['boilerplate_code'] = boiler # Try to tokenize extracted code and boilerplate. try: toks, types = tokenize_and_templatize(extracted) target_cell['extracted_code_tokens'] = replace_newlines_indents(toks, types, enable_assert=False, comments=True, strings=True) target_cell['extracted_code_types'] = types toks, types = tokenize_and_templatize(boiler) boiler_toks = replace_newlines_indents(toks, types, enable_assert=False, comments=True, strings=True) # If trivial just leave boilerplate tokens empty. if is_trivial_boilerplate(boiler_toks): boiler_toks = [] types = [] target_cell['boilerplate_code_tokens'] = boiler_toks target_cell['boilerplate_code_types'] = types return target_cell except: return 'not tokenizable autograded' else: return 'multiple insertion points' else: return 'no boilerplate' return 'only boiler'